Пример #1
0
exports = module.exports = function(resultSet, callback) {
  //
  // Stores orders
  //

  if(resultSet.objects.length === 0) {

    // SKIP EMPTY MESSAGES
    return callback(null, resultSet);

  } else {

    // Retry 3X by default, since the CTE-upsert-code is not very safe for parallel execution
    async.retry(3, executeQuery.bind(null, resultSet), function(err) {

      // Handle errors
      if (err) {
        return callback(err, null);
      } else {
        callback(null, resultSet);
      }
    });

  }
};
Пример #2
0
GeoController.prototype._getAddressFromKakao = function (callback) {
    var that = this;
    var index = 0;

    async.retry(kakaoKeys.length,
        function (cb) {
            let url = 'https://dapi.kakao.com/v2/local/geo/coord2regioncode.json';
            url += '?x='+that.lon;
            url += '&y='+that.lat;
            url += '&input_coord=WGS84';
            let header = {
                Authorization: 'KakaoAK ' + kakaoKeys[index]
            };

            index++;

            log.info(url);
            that.axios.get(url, {headers: header})
            .then(response=>{
                return cb(null, response.data);
            })
            .catch(e=>{
                return cb(e);
            });
        },
        function (err, result) {
            if (err) {
                return callback(err);
            }
            callback(null, result);
        });

    return this;
};
Пример #3
0
BlockService.prototype.onHeaders = function(callback) {

  var self = this;

  if (self._pauseSync) {
    log.warn('Block Service: pausing sync due to config option.');
    return callback();
  }

  // if this service is waiting on block-related callbacks to be fired in the event loop,
  // then we need to wait for the _processingBlock flag to be set to false.
  // when this flag is false, we know we aren't waiting on any new blocks or historical blocks
  // that we asked for, but not yet received
  self._initialSync = true;

  // a heavy block could take a really long time to index
  async.retry({ interval: 1000, times: 100 }, function(next) {
    return next(self._processingBlock);
  }, function(err) {
    if (err) {
      return callback(err);
    }
    self._onHeaders(callback);
  });

};
Пример #4
0
exports.work = (from, size) => {
    let source = drivers.get(exports.env.options.drivers.source).driver;

    function get(callback) {
        source.getData(exports.env, (err, data) => {
            if (err) {
                return callback(err);
            }
            // TODO validate data format
            // TODO validate that data.length == size and throw a warning if not (does this work in a cluster?)
            if (!data || !data.length) {
                exports.send.end();
            } else if (exports.env.options.run.test) {
                exports.send.done(data.length);
            } else {
                exports.storeData(data);
            }
            callback();
        }, from, size);
    }

    async.retry(exports.env.options.errors.retry, callback => exports.waitOnTargetDriver(get, callback), (err) => {
        if (err) {
            if (exports.env.options.errors.ignore) {
                exports.send.done(size);
            } else {
                exports.send.error(err);
            }
        }
    });
};
Пример #5
0
GeoController.prototype._getAddressFromGoogle = function (callback) {
    var that = this;
    //retry with key
    var url = that.googleUrl+"/maps/api/geocode/json?latlng=" + that.lat + "," + that.lon;
    if (that.lang) {
        url += "&language="+that.lang;
    }
    if (googleApiKey) {
        url += "&key="+googleApiKey;
    }
    else {
        log.warn("google api key is not valid");
    }
    log.info(url);
    async.retry(3,
        function (cb) {
            that._request(url, function (err, result) {
                if (err) {
                    return cb(err);
                }
                cb(null, result);
            });
        },
        function (err, result) {
            if (err) {
                return callback(err);
            }
            callback(null, result);
        });

    return this;
};
Пример #6
0
exports.resetTarget = (results, callback) => {
    async.retry(exports.env.options.errors.retry, callback => {
        log.debug('Resetting target driver to begin operations');
        let target = drivers.get(exports.env.options.drivers.target).driver;
        target.reset(exports.env, callback);
    }, callback);
};
Пример #7
0
exports.resetSource = (results, callback) => {
    async.retry(exports.env.options.errors.retry, callback => {
        log.debug('Resetting source driver to begin operations');
        let source = drivers.get(exports.env.options.drivers.source).driver;
        source.reset(exports.env, callback);
    }, callback);
};
Пример #8
0
function getResource(url, options, callback) {
  if (_.isFunction(options)) {
    callback = options;
    options = {};
  }

  options.method = 'GET';
  options.gzip = true;
  options.url = url;
  async.retry({}, function (asyncCallback) {
    new Request(options, function(err, response, data) {
      if (err)
        return asyncCallback(new Error('Can not GET "' + url +'": ' + err));
      if (response.statusCode !== 200)
        return asyncCallback(new Error('Can not GET "' + url +'": ' + response.statusMessage));
      asyncCallback(null, {response: response, data: data});
    });
  }, function (err, result) {
    if (err)
      return callback(err);

    console.log(url);
    callback(null, result.response, result.data);
  });
}
Пример #9
0
			async.eachLimit(urls, argv.concurrency, function(url, callback) {
				async.retry(argv.retries, function(callback) {
					var start = (new Date()).getTime();
					request({
						method: argv.method,
						url: url,
						headers: headers
					}, function(err, res, body) {
						if (err) return callback(err);

						var time = (new Date()).getTime() - start;
						var statuscolor = res.statusCode !== 200 ? 'red' : 'green';
						var size_data = filesize(res.body.length);
						var size_length = res.headers['content-length'] ? filesize(Number(res.headers['content-length'])) : '(no content-length)';

						process.stdout.cursorTo(0);
						console.log(chalk.gray('[') + chalk[statuscolor](res.statusCode) + chalk.grey(']') + ' ' + url + ' ' + chalk.blue(time + 'ms') + ' ' + chalk.grey(size_data + ', ' + size_length));
						bar.tick();

						if (res.statusCode !== 200) {
							count_failed++;
							callback('Request failed (non-200 status)');
						} else {
							count_succeeded++;
							callback();
						}
					});
				}, function(err) {
					if (err && argv.allowfailures) err = null;
					callback(err);
				});
			}, callback);
Пример #10
0
 function(cb) {
   var results = {};
   FIELDS.forEach(function(field) {
     results[field.name] = options[field.name]
   });
   async.retry(3,
     function(cb) {
       cli.requireAnswers(FIELDS, results, function(results) {
         validateAccount(results, options, function(err) {
           if (err) {
             if (err.message === 'bad_auth') {
               emit('Authorization failed. Please retry username & password.');
               cli.updateDefaultValue(FIELDS, 'username', results.username);
               delete(results.username);
               delete(results.password);
             } else if (err.message === 'bad_url') {
               emit('Failed to connect to API. Please check uri.');
               cli.updateDefaultValue('baseuri', 'username', results.baseuri);
               delete(results.baseuri);
             }
           }
           cb(err, results);
         });
       });
     },
     cb);
 }
Пример #11
0
    getConfigInfo: function (progress, data, callback) {
      debug('in getConfigInfo', data);

      var sendReceive = function(cb) {
        otu2CommandResponse(DMDateTime, progress, function(err, result){
          if (err) {
            debug('Failure trying to talk to device.');
            debug(err);
            debug(result);
            cb(err, null);
          } else {

            var str = String.fromCharCode.apply(null, result.bytes);

            var pattern = /F "(\w+)","(\d{2}\/\d{2}\/\d{2})","(\d{2}:\d{2}:\d{2})   " (\w+)/;
            var toMatch = pattern.exec(str);

            if (toMatch == null) {
              return cb(new Error('Unexpected response from meter.'), null);
            }

            var currentDeviceTime = sundial.formatDeviceTime(buildDateTime(toMatch[2],  toMatch[3]));
            debug('Device date/time:', currentDeviceTime);
            common.checkDeviceTime(currentDeviceTime, cfg, function(err) {
              progress(100);
              return cb(err, data);
            });
          }
        });
      };

      async.retry({ times: RETRIES, interval: TIME_TO_WAIT }, sendReceive, function(err, result) {
        callback(err, result);
      });
    },
Пример #12
0
    fetchData: function (progress, data, cb) {
      debug('in fetchData');

      var sendReceive = function(callback) {
        otu2CommandResponse(DMUpload, progress, function(err, result){
          if (err) {
            debug('Failure trying to talk to device.');
            debug(err);
            callback(err, null);
          } else {
            debug('fetchData otu2CommandResponse:', result);
            _.assign(data, result);

            prepBGData(progress, data, function(err, result) {
              data.post_records = result;

              if (err) {
                debug(err);
                return callback(err, null);
              }
              if (data.post_records === null) {
                return callback(new Error('Received invalid data from meter.'), null);
              }

              callback(null, data);
            });
          }
        });
      };

      async.retry({ times: RETRIES, interval: TIME_TO_WAIT }, sendReceive, function(err, result) {
        cb(err, result);
      });
    },
Пример #13
0
var waitForBlocksGenerated = function(callback) {

  var httpOpts = {
    hostname: 'localhost',
    port: 53001,
    path: '/api/status',
    method: 'GET',
    headers: {
      'Content-Type': 'application/json'
    }
  };

  async.retry({ interval: 1000, times: 100 }, function(next) {

    request(httpOpts, function(err, data) {
      if (err) {
        return next(err);
      }
      if (data.info.blocks !== blocksGenerated) {
        return next(data);
      }
      next();
    });

  }, callback);
};
Пример #14
0
module.exports.run = function() {
    var connString = 'postgres://' + config.PG_USER + ':' + config.PG_PASS +
                     '@' + config.PG_HOST;

    var connectionDone;

    async.retry(
        5,
        function(cb) {
            pg.connect(connString, function(err, client, done) {
                if (!client) {
                    console.log('Waiting for connection ...');
                    setTimeout(function() { cb(err); }, 5000);
                } else {
                    connectionDone = done;
                    cb(null, client);
                }
            });
        },
        function(err, client) {
            if (err) { return console.error(err); }
            client.query('DROP DATABASE IF EXISTS alc_test', function() {
                client.query('CREATE DATABASE alc_test', function(err) {
                    if (err) { console.error(err); }
                    console.log('Recreated test database');
                    done();
                });
            });
        }
    );
}
Пример #15
0
 // check the customization status repeatedly until it's available
 function (next) {
     var options = extend({
         interval: 5000,
         times: 30
     }, params);
     options.errorFilter = function (err) {
         // if it's a timeout error, then getCorpora is called again after params.interval
         // otherwise the error is passed back to the user
         // if the params.times limit is reached, the error will be passed to the user regardless
         return err.code === SpeechToTextV1.ERR_TIMEOUT;
     };
     async.retry(options, function (done) {
         self.getCorpora(params, function (err, corpora) {
             if (err) {
                 done(err);
             }
             else if (isProcessing(corpora)) {
                 // if the loop times out, async returns the last error, which will be this one.
                 err = new Error('Corpora is still being processed, try increasing interval or times params');
                 err.code = SpeechToTextV1.ERR_TIMEOUT;
                 done(err);
             }
             else if (isAnalyzed(corpora)) {
                 done(null, corpora);
             }
             else {
                 done(new Error('Unexpected corpus analysis status'));
             }
         });
     }, next);
 }
Пример #16
0
        table.insert(data, function(err, insertErrors) {
          assert.ifError(err);

          if (insertErrors.length > 0) {
            done(insertErrors[0].errors[0]);
            return;
          }

          function query(callback) {
            var row;

            table.query('SELECT * FROM ' + table.id + ' WHERE id = ' + data.id)
              .on('error', callback)
              .once('data', function(row_) { row = row_; })
              .on('end', function() {
                if (!row) {
                  callback(new Error('Query returned 0 results.'));
                  return;
                }

                assert.strictEqual(row.name, data.name);
                assert.strictEqual(row.breed, data.breed);
                assert.strictEqual(row.id, data.id);
                assert.deepEqual(row.dob, now);
                callback();
              });
          }

          async.retry({ times: 3, interval: 2000 }, query, done);
        });
Пример #17
0
GeoController.prototype._getAddressFromDaum = function (callback) {
    var that = this;
    var index = 0;

    async.retry(daumKeys.length,
        function (cb) {
            var url = that.daumUrl+'/local/geo/coord2addr'+
                '?apikey=' + daumKeys[index] +
                '&longitude='+ that.lon +
                '&latitude='+that.lat+
                '&inputCoordSystem=WGS84'+
                '&output=json';
            index++;

            log.info(url);
            that._request(url, function (err, result) {
                if(err) {
                    return cb(err);
                }
                cb(null, result);
            });
        },
        function (err, result) {
            if (err) {
                return callback(err);
            }
            callback(null, result);
        });

    return this;
};
Пример #18
0
/**
 * Create new room if needed
 *
 * @this HotelClerk
 * @param {String} index
 * @param {String} collection
 * @param {Object} filters
 * @returns {Promise} promise
 */
function createRoom (index, collection, filters) {
  var
    diff,
    deferred,
    roomId;

  if (!index || !collection) {
    return q.reject(new BadRequestError('Cannot subscribe without an index and a collection'));
  }

  roomId = this.kuzzle.dsl.createFilterId(index, collection, filters);
  deferred = q.defer();

  async.retry(callback => {
    // if the room is about to be destroyed, we have to delay its re-creation until its destruction has completed
    if (this.rooms[roomId] && this.rooms[roomId].destroyed) {
      return callback(new InternalError('Cannot create the room ' + roomId + ' because it has been marked for destruction'));
    }

    if (!this.rooms[roomId]) {
      // If it's a new room, we have to calculate filters to apply on the future documents
      this.kuzzle.dsl.register(roomId, index, collection, filters)
        .then(response => {
          var formattedFilters;

          if (response && response.filter !== undefined) {
            formattedFilters = response.filter;
          }

          if (response) {
            diff = response.diff;
          }

          if (this.rooms[roomId]) {
            return callback(null, roomId);
          }

          this.kuzzle.pluginsManager.trigger('room:new', {roomId: roomId, index: index, collection: collection, formattedFilters: formattedFilters})
            .then(modifiedData => {
              this.rooms[modifiedData.roomId] = {
                id: modifiedData.roomId,
                customers: [],
                index: modifiedData.index,
                channels: {},
                collection: modifiedData.collection
              };

              callback(null, { diff, roomId });
            });
        })
        .catch(error => callback(error));
    }
    else {
      callback(null, { roomId });
    }
  }, deferred.makeNodeResolver());

  return deferred.promise;
}
Пример #19
0
    value: function _tryPutBlock(block, times, cb) {
      var _this3 = this;

      log('trying to put block %s', block.data.toString());
      async.retry({ times: times, interval: 400 }, function (done) {
        _this3.datastore.put(block, done);
      }, cb);
    }
Пример #20
0
function writeSpec(source, type, exPatch, callback) {
  var getSpecTask = converter.getSpec.bind(this, source, type);
  async.retry({}, getSpecTask, function (err, spec) {
    assert(!err, err);

    var fixup = readYaml(getOriginFixupPath(spec));
    jsondiffpatch.patch(spec, fixup);

    convertToSwagger(spec, function (error, swagger) {
      var result = {
        spec: spec,
        errors: error
      };

      if (error)
        return callback(error, result);

      try {
        patchSwagger(swagger, exPatch);
      }
      catch (e) {
        callback(e, result);
      }

      expandPathTemplates(swagger);
      replaceSpacesInSchemaNames(swagger);

      result.swagger = swagger;

      function done(errors, warnings) {
        result.warnings = warnings;

        if (errors)
          return callback(errors, result);

        if (warnings)
          logYaml(warnings);

        saveSwagger(swagger);
        callback(null, result);
      }

      function validateAndFix() {
        validateSwagger(swagger, function (errors, warnings) {
          if (!_.isArray(errors))
            return done(errors, warnings);

          if (fixSpec(swagger, errors))
            validateAndFix();
          else
            validateSwagger(swagger, done);
        });
      }

      validateAndFix();
    });
  });
}
  async.mapLimit(_.clone(options.airlines, true), 20, function (airline, callback) {

    airline.baseDir = options.baseDir;
    airline.save = options.save || false;
    async.retry(5, function (callback) {
      getDestinations(airline, callback);
    }, callback);

  }, function (err, airlines) {
Пример #22
0
            imageUrl: function(callback) {
                if (args) {
                    return callback(null, args);
                }

                async.retry(function(callback) {
                    util.callbackify(randomPuppy)(callback);
                }, callback);
            },
Пример #23
0
exports.getTargetStatistics = (results, callback) => {
    async.retry(exports.env.options.errors.retry, callback => {
        log.debug('Fetching target statistics before starting run');
        let target = drivers.get(exports.env.options.drivers.target).driver;
        target.getTargetStats(exports.env, (err, targetStats) => {
            exports.env.statistics.target = util._extend(exports.env.statistics.target, targetStats);
            callback(err);
        });
    }, callback);
};
Пример #24
0
exports.getSourceStatistics = (results, callback) => {
    async.retry(exports.env.options.errors.retry, callback => {
        log.debug('Fetching source statistics before starting run');
        let source = drivers.get(exports.env.options.drivers.source).driver;
        source.getSourceStats(exports.env, (err, sourceStats) => {
            exports.env.statistics.source = util._extend(exports.env.statistics.source, sourceStats);
            callback(err);
        });
    }, callback);
};
Пример #25
0
 async.map(serviceList, (serviceName, next) => {
   async.retry({times: 10, interval: 200}, (next) => {
     try {
       var node = nodes[serviceName].next()
       next(null, node)
     } catch (e) {
       next(e);
     }
   }, next)
 }, (err, results) => next.apply(null, results))
Пример #26
0
 function(next) {
   async.retry({times: 100, interval: 10},
     function(cb){
       setEdge(pinForSetup, edge, cb);
     },
     function(err){
       // wrapped here because waterfall can't handle positive result
       next(err);
     });
 },
Пример #27
0
internals.Client.prototype._makeAPICall = function (settings, callback) {

    const options = {
        method: settings.method,
        url: this.options.baseUri + settings.endpoint,
        headers: {
            'Authorization': 'Bearer ' + this.apiKey,
            'User-Agent': 'node-client v' + require('../version')
        },
        qs: settings.query || {},
        json: settings.payload || true,
        timeout: this.options.timeout,
        agent: this.options.agent
    };

    const strategy = {
        times: (this.options.maxAttempts < 1) ? 1 : this.options.maxAttempts,
        retryDelay: this.options.retryDelay
    };

    // console.log('options', options);
    // console.log('strategy', strategy);

    Async.retry(strategy, (done, previousResult) => {

        Request(options, (error, response, body) => {

            // console.log('error', error);
            if (error) {
                return done(error);
            }

            if (response && response.statusCode >= 400) {
                let message = response.statusCode + ' ' + body;

                if (body.error) {
                    message = body.error;
                }

                if (body.error && body.message) {
                    message += ' - ' + body.message;
                }

                return done(new Error(message));
            }

            done(null, body);
        });

    }, callback);
};
    checkIfIndexExists (callback)
    {
        const self = this;

        const tryToConnect = function (callback)
        {
            self.ensureElasticSearchIsReady(function (err)
            {
                if (isNull(err))
                {
                    self.client.indices.exists(
                        {
                            index: self.short_name
                        },
                        function (err, exists)
                        {
                            callback(err, exists);
                        }
                    );
                }
                else
                {
                    callback(err, "Unable to connect to elasticsearch for checking is index exists");
                }
            });
        };

        // try calling apiMethod 10 times with linear backoff
        // (i.e. intervals of 100, 200, 400, 800, 1600, ... milliseconds)
        async.retry({
            times: 240,
            interval: function (retryCount)
            {
                const msecs = 1000;
                Logger.log("debug", "Waiting " + msecs / 1000 + " seconds to verify if ElasticSearch index " + self.short_name + " exists...");
                return msecs;
            }
        }, tryToConnect, function (err, result)
        {
            if (isNull(err))
            {
                callback(err, Boolean(result));
            }
            else
            {
                const msg = "Unable to verify if ElasticSearch index " + self.short_name + " exists, even after several retries. This is a fatal error.";
                Logger.log("error", result);
                throw new Error(msg);
            }
        });
    }
Пример #29
0
 _wait(options, cb) {
   const interval = 1000;
   const times = Math.ceil(options.timeout / interval);
   async.retry({ times: times, interval: interval }, next => {
     this.get(options.name, (err, result) => {
       if (err) {
         if (err.code === 404) return next(null);
         return next(err);
       }
       if (result.metadata.uid !== options.uid) return next(null);
       next(new Error('Waiting for namespace removal'));
     });
   }, cb);
 }
Пример #30
0
 async.eachSeries(columns, function (column, __done) {
   process.stdout.write('        Creating column ' + column + '...');
   async.retry(NumRetries, function (___done) {
     scripty.invoke('mobile table update --addColumn ' + column + ' ' + nconf.get('name') + ' ' + table.name, function (err, results) {
       if (!err) {
         console.log(' OK'.green.bold);
       } else if (resource_already_exists(err)) {
         console.log(' column already exists, OK'.yellow.bold);
         err = null;
       }
       ___done(err);
     });
   }, __done);
 }, _done);