trjhtr

I need to retrieve a file from a website every minute that contains boats location.

Then, I parse the file, delete all the boats from my database and re-create all of them at the new position indicated in the file.

Here's my code so far - in the server.js of my loopbackjs app:

var CronJob = require('cron').CronJob;
var exec = require("child_process").exec;
var reload = require('require-reload')(require);
var app = module.exports = loopback();

// Bootstrap the application, configure models, datasources and middleware.(loopbackjs mandatory code)
boot(app, __dirname, function (err) 
 if (err)
 throw err;

 var Boat = app.models.Boat;
 var ais_url_france = 'http://data.aishub.net/ws.php?username=XXXXXXXXX&format=1&output=json&compress=0&latmin=40&latmax=52&lonmin=-5&lonmax=10';

 //start the cronjob on a node env every min
 new CronJob('* * * * *', function () 
 // get the file and save as a .json
 exec('wget "' + ais_url_france + '" -O aishub_france.json', function (error, stdout, stderr) 
 // I need to use reload here because the file is stored in memory otherwise and only read once after the first download...
 var file = reload('../aishub_france.json');
 if (file[0].ERROR === false) 
 var boats = file[1];
 if (boats.length > 0) 
 Boat.destroyAll(typeId: 'aishub_boats', function (err, info) 
 for (var i = 0; i < boats.length; i++) 
 Boat.create(
 name: boats[i].NAME,
 latlng: 
 lat: boats[i].LATITUDE,
 lng: boats[i].LONGITUDE
 ,
 typeId: 'aishub_boats'
 );
 
 );
 
 
 );
 , null, true, null);
);

I'm not sure the code here is so great but here are my troubles:

• The downloaded file is huge (1,3Mo)


• I need to remove and re-add all the boats because they change location and some might appear/disappear (maybe there is a better solution)


• Not sure using require-reload is the best solution, but if I don't the file is read only once in the cronjob even if the donwloaded file changed each time...


• 
  

  I think I might have some memory leak because I'm having strange stacks trace recently after I added this code. Here is the trace:

  
  
  

  33503717 ms: Mark-sweep 1288.0 (1403.6) -> 1288.0 (1404.6) MB, 672.5 / 0.0 ms [allocation failure] [GC in old space requested].

  
  
  

  33504750 ms: Mark-sweep 1288.0 (1404.6) -> 1288.0 (1404.6) MB, 847.3 / 0.0 ms [allocation failure] [GC in old space requested].

  
  
  

  33505597 ms: Mark-sweep 1288.0 (1404.6) -> 1288.0 (1403.6) MB, 846.4 / 0.0 ms [last resort gc].

  
  
  

  33506438 ms: Mark-sweep 1288.0 (1403.6) -> 1288.0 (1403.6) MB, 841.4 / 0.0 ms [last resort gc].

  
  

OK, thank you James!

I was able to get rid of require-reload using request, and use the compression with adm-zip on the fly:

var CronJob = require('cron').CronJob;
var request = require('request');
var AdmZip = require('adm-zip');

var ais_url_france = 'http://data.aishub.net/ws.php?username=XXX&format=1&output=json&compress=1&latmin=40&latmax=52&lonmin=-5&lonmax=10';
new CronJob('* * * * *', function () 
 request(url: ais_url_france, encoding: null, function (error, response, data) 
 if (!error && response.statusCode === 200) 
 var zip = new AdmZip(data);
 var zipEntries = zip.getEntries();
 if (zipEntries && zipEntries.length) 
 var file = JSON.parse(zip.readAsText(zipEntries[0]));
 if (file[0].ERROR === false) 
 var boats = file[1];
 if (boats.length > 0) 
 Boat.destroyAll(function (err, info) 
 for (var i = 0; i < boats.length; i++) 
 Boat.create(
 name: boats[i].NAME,
 latlng: 
 lat: boats[i].LATITUDE,
 lng: boats[i].LONGITUDE
 
 );
 
 );
 
 
 
 
 );
 , null, true, null);