diff --git a/downloader.js b/downloader.js new file mode 100644 index 0000000..2727216 --- /dev/null +++ b/downloader.js @@ -0,0 +1,32 @@ +const axios = require('axios'); +const fs = require('fs').promises; + +/** + * Kevin Pan | pan261@purdue.edu | Last Modified: 11/22/2021 + * + * Downloads a file by making a GET request on the storageUrl and then + * saves the file as `filename` under the `destination` directory + * @param {String} storageUrl + * @param {String} fileName + * @param {String} destination + * @param {Object} credentials + */ +exports.download = async (storageUrl, fileName, destination, credentials) => { + await axios({ + method: 'GET', + url: storageUrl, + headers: { + Authorization: `Bearer ${credentials.access_token}` + } + }) + .then(async res => { + fs.writeFile(`./${destination}/${fileName}`, res.data).then(() => { + console.log("finished downloading", fileName); + }); + }) + .catch(err => { + console.log("error while processing " + fileName); + // console.log(err); + }); +} + diff --git a/taskRunner.js b/taskRunner.js index 47e7144..ac29f2b 100644 --- a/taskRunner.js +++ b/taskRunner.js @@ -1,5 +1,6 @@ const axios = require('axios'); const watcher = require('./watcher'); +const downloader = require('./downloader'); require('dotenv').config(); /** @@ -14,6 +15,8 @@ require('dotenv').config(); // TODO const projectID = "" +// TODO: create map from cloud directories to local + // parse command line arguments from docker-compose to get folder ID const folderIdMap = { gantry: "GANTRY_FOLDER_ID", @@ -88,7 +91,7 @@ const getCredentials = () => { /** * Function to dispatch the watcher */ -const dispatchWatcher = (interval) => { +const dispatchWatcher = async (interval) => { if (credentials === null) { console.log( '\x1b[93mtaskRunner.js::dispatchWatcher:', @@ -96,12 +99,23 @@ const dispatchWatcher = (interval) => { ); return; } + + const items = await watcher.watch(adminProjectID, adminProjectTopFolder, credentials, interval); + + if (Object.keys(items).length == 0) { + return; // no items to download + } - watcher.watch(adminProjectID, adminProjectTopFolder, credentials, interval); + for (index in items) { + // dispatch the downloader here + console.log(items[index].name); + downloader.download(items[index].url, items[index].name, 'download_tests', credentials); + } }; getCredentials(); // There should be a buffer between how far back the watcher checks // and how often it runs. We may need to test this out. -setInterval(dispatchWatcher, 5000, 7000); \ No newline at end of file +setInterval(dispatchWatcher, 5000); + diff --git a/watcher.js b/watcher.js index 1b78f7a..1cdae5b 100644 --- a/watcher.js +++ b/watcher.js @@ -10,6 +10,8 @@ const axios = require('axios'); * @param {Object} credentials credentials for the call * @param {Integer} interval number of ms used to filter the date * if an interval is not passed, watcher will return all files in the given folder + * + * @return {Object} changedItems array of items changed that need to be downloaded */ exports.watch = (projectID, folderID, credentials, interval) => { let timeFilter = {}; @@ -19,10 +21,12 @@ exports.watch = (projectID, folderID, credentials, interval) => { var dateXSecondsAgo = new Date(dateNow.getTime() - interval); timeFilter = { 'filter[lastModifiedTime]-ge': dateXSecondsAgo.toISOString() }; } + console.log( '\x1b[96mwatcher.js::watch:', '\x1b[0mChecking for files modified since ' + interval/1000 + ' sec ago'); - axios({ + + const items = axios({ method: 'GET', url: `https://developer.api.autodesk.com/data/v1/projects/${projectID}/folders/${folderID}/contents`, headers: { @@ -41,16 +45,33 @@ exports.watch = (projectID, folderID, credentials, interval) => { return; } + let changedItems = {}; + for (var index in data) { // console.log(data[index]); const displayName = data[index].attributes.displayName; + const name = data[index].attributes.name; + var saveName = displayName; + + if (displayName.lastIndexOf('.') != name.lastIndexOf('.')) { + saveName += name.substring(name.lastIndexOf('.')); + } + const lastModifiedTime = data[index].attributes.lastModifiedTime; + const storageUrl = data[index].relationships.storage.meta.link.href; console.log('\x1b[92mwatcher.js::watch: \x1b[0m' + displayName + ' was modified at ' + lastModifiedTime); // TODO: if item is not in db, add to db and then download - // TODO: if item is in db, check and compare lastModifiedTime + // TODO: if item is in db, check and compare lastModifiedTime then download those + // where time does not match with db time + + // index will have to change because not all items may need to be added to this list + changedItems[index] = {name: saveName, url: storageUrl}; + // console.log(changedItems); } + + return changedItems; }) .catch(err => { console.log( @@ -58,4 +79,6 @@ exports.watch = (projectID, folderID, credentials, interval) => { '\x1b[0mError accessing Forge folder contents API' ); }); + + return items; };