diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..cb82aea --- /dev/null +++ b/.dockerignore @@ -0,0 +1,4 @@ +# Set to ignore everything except /src and package.json +* +!src +!package.json \ No newline at end of file diff --git a/.gitignore b/.gitignore index c93b551..fc4d205 100644 --- a/.gitignore +++ b/.gitignore @@ -6,7 +6,10 @@ node_modules credentials.json download_tests build + FileShare upload/ lathes/ -mills/ \ No newline at end of file +mills/ +file_share +postgres/db_data diff --git a/docker-compose.yml b/docker-compose.yml index 87c052f..cf03a0e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,63 +1,33 @@ version: "3.8" services: + + # access through `psql -h localhost -p 5432 -d filesync -U user`, password is 'pass' + # to be able to build after the initializing the db, you may need to run `sudo chown -R $USER db_data/` + database: + image: 'postgres:13' + env_file: ./.env + volumes: + - ./postgres/db_data/:/var/lib/postgresql/data/ + - ./postgres/init/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql + ports: + - ${PORT_DB}:${PORT_DB} + server: build: context: . dockerfile: Dockerfile + depends_on: + - database env_file: ./.env volumes: - ./src/build:/ffs/ - /ffs/node_modules - - ./FileShare:/ffs/FileShare/ + - ./file_share:/ffs/file_share/ + - ${LOCAL_DIR_GANTRY}:/ffs/file_share/Gantry/ + - ${LOCAL_DIR_MILL}:/ffs/file_share/Mill/ + - ${LOCAL_DIR_LATHE}:/ffs/file_share/Lathe/ + - ${LOCAL_DIR_WATERJET}:/ffs/file_share/Waterjet/ ports: - - ${PORT}:${PORT} - command: "node server.js" - - # task_runner_gantry: - # build: - # context: . - # dockerfile: Dockerfile - # env_file: ./.env - # volumes: - # - ./src/build:/ffs/ - # - /ffs/node_modules - # - ${LOCAL_DIR_GANTRY}:/ffs/FileShare/Gantry - # network_mode: host - # command: "node taskRunner.js gantry" - # - # task_runner_lathe: - # build: - # context: . - # dockerfile: Dockerfile - # env_file: ./.env - # volumes: - # - ./src/build:/ffs/ - # - /ffs/node_modules - # - ${LOCAL_DIR_LATHE}:/ffs/FileShare/Lathe - # network_mode: host - # command: "node taskRunner.js lathe" - - # task_runner_mill: - # build: - # context: . - # dockerfile: Dockerfile - # env_file: ./.env - # volumes: - # - ./src/build:/ffs/ - # - /ffs/node_modules - # - ${LOCAL_DIR_MILL}:/ffs/FileShare/Mill - # network_mode: host - # command: "node taskRunner.js mill" - - # task_runner_waterjet: - # build: - # context: . - # dockerfile: Dockerfile - # env_file: ./.env - # volumes: - # - ./src/build:/ffs/ - # - /ffs/node_modules - # - ${LOCAL_DIR_WATERJET}:/ffs/FileShare/Waterjet - # network_mode: host - # command: "node taskRunner.js waterjet" \ No newline at end of file + - ${PORT_SERVER}:${PORT_SERVER} + command: "node server.js" \ No newline at end of file diff --git a/environment.d.ts b/environment.d.ts index 507be66..e91e6ca 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -4,7 +4,9 @@ declare global { FORGE_CLIENT_ID: string; FORGE_CLIENT_SECRET: string; FORGE_CALLBACK_URL: string; - PORT: string; + PORT_SERVER: number; + PORT_DB: number; + WEBHOOK_TOKEN: string; } } } diff --git a/package-lock.json b/package-lock.json index 2136d8b..8c45d16 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,9 @@ "crypto": "^1.0.1", "dotenv": "^10.0.0", "express": "^4.17.1", - "forge-apis": "^0.8.6" + "forge-apis": "^0.8.6", + "generic-pool": "^3.8.2", + "ts-postgres": "^1.2.1" }, "devDependencies": { "@types/dotenv": "^8.2.0", @@ -1714,6 +1716,14 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "node_modules/generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -3222,6 +3232,22 @@ "node": ">=0.8" } }, + "node_modules/ts-postgres": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ts-postgres/-/ts-postgres-1.2.1.tgz", + "integrity": "sha512-5cCIdGMMlX8g+cFs0DFykLKyTmPvJ9gCv5TR13R90Azz8gzEtugYq6gKm5glBXF66gtvsP/lUashU5akfQsFaA==", + "dependencies": { + "ts-typed-events": "^2.0.1" + }, + "engines": { + "node": ">=10.7.0" + } + }, + "node_modules/ts-typed-events": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ts-typed-events/-/ts-typed-events-2.0.1.tgz", + "integrity": "sha512-7V+vCn52mZVSe+udf0L2SRkvs8BquF7xOtVktJsiyz3IGgcAhHwp3aetPcrxT5Xq23Bb/tBX0azQ6BBvPVytZg==" + }, "node_modules/tsconfig-paths": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", @@ -4751,6 +4777,11 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==" + }, "get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -5863,6 +5894,19 @@ "punycode": "^2.1.1" } }, + "ts-postgres": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ts-postgres/-/ts-postgres-1.2.1.tgz", + "integrity": "sha512-5cCIdGMMlX8g+cFs0DFykLKyTmPvJ9gCv5TR13R90Azz8gzEtugYq6gKm5glBXF66gtvsP/lUashU5akfQsFaA==", + "requires": { + "ts-typed-events": "^2.0.1" + } + }, + "ts-typed-events": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ts-typed-events/-/ts-typed-events-2.0.1.tgz", + "integrity": "sha512-7V+vCn52mZVSe+udf0L2SRkvs8BquF7xOtVktJsiyz3IGgcAhHwp3aetPcrxT5Xq23Bb/tBX0azQ6BBvPVytZg==" + }, "tsconfig-paths": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", diff --git a/package.json b/package.json index 31f7a2c..b2779b7 100644 --- a/package.json +++ b/package.json @@ -24,8 +24,10 @@ "cookie-session": "^1.4.0", "crypto": "^1.0.1", "dotenv": "^10.0.0", - "express": "^4.17.2", - "forge-apis": "^0.8.6" + "express": "^4.17.1", + "forge-apis": "^0.8.6", + "generic-pool": "^3.8.2", + "ts-postgres": "^1.2.1" }, "devDependencies": { "@types/dotenv": "^8.2.0", diff --git a/postgres/init/create_tables.sql b/postgres/init/create_tables.sql new file mode 100644 index 0000000..2e09be1 --- /dev/null +++ b/postgres/init/create_tables.sql @@ -0,0 +1,29 @@ +-- Primary keys must include folder name in order to avoid collisions +-- between filenames in different folders + +CREATE TABLE IF NOT EXISTS fusion ( + fusion_id varchar(250) NOT NULL, + folder_name varchar(250) NOT NULL, + file_name varchar(250) NOT NULL, + username varchar(250), + size int, + version int, + PRIMARY KEY (fusion_id) +); + +CREATE TABLE IF NOT EXISTS local ( + file_name varchar(250) NOT NULL, + folder_name varchar(250) NOT NULL, + fusion_id varchar(250), + PRIMARY KEY (file_name, folder_name) +); + +CREATE TABLE IF NOT EXISTS archive ( + fusion_id varchar(250) NOT NULL, + file_name varchar(250) NOT NULL, + folder_name varchar(250) NOT NULL, + username varchar(250) NOT NULL, + size int, + version int, + PRIMARY KEY (fusion_id, folder_name) +); \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index 19eb40f..91b6855 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,9 +3,7 @@ */ import { AuthClientThreeLegged } from 'forge-apis'; - import * as dotenv from 'dotenv'; - dotenv.config({path: '../.env'}); export const ForgeAuthClient = AuthClientThreeLegged; @@ -53,9 +51,13 @@ export const folderIDtoLocal: any = { "urn:adsk.wipprod:fs.folder:co.vJLXAKGbQQayeljr-nwztQ": "/Waterjet" } -export const port = process.env.PORT; +export const port = process.env.PORT_SERVER; export const forgeClientId = process.env.FORGE_CLIENT_ID; export const forgeClientSecret = process.env.FORGE_CLIENT_SECRET; export const forgeCallbackURL = process.env.FORGE_CALLBACK_URL; export const hookCallbackURL = `${process.env.HOOK_CALLBACK_HOSTNAME}/hook`; -export const webhookToken: any = process.env.WEBHOOK_TOKEN; \ No newline at end of file +export const webhookToken = process.env.WEBHOOK_TOKEN; +export const postgresUser = process.env.POSTGRES_USER; +export const postgresPassword = process.env.POSTGRES_PASSWORD; +export const postgresPort = process.env.PORT_DB; +export const postgresDB = process.env.POSTGRES_DB; \ No newline at end of file diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 0000000..3ba3686 --- /dev/null +++ b/src/db.ts @@ -0,0 +1,108 @@ +import { Client, Query } from 'ts-postgres'; +import { createPool } from 'generic-pool'; +import * as config from './config'; + +const pool = createPool({ + create: async () => { + return new Promise((resolve, reject) => { + + const client = new Client({ + 'host': 'database', // name of docker container, not 'localhost' + 'port': config.postgresPort, + 'user': config.postgresUser, + 'password': config.postgresPassword, + 'database': config.postgresDB + }); + + client.connect() + .then(() => { + console.log("successfully connected to db") + resolve(client); + }).catch((err: any) => { + console.log() + console.log(err) + reject(err); + }); + }); + }, + destroy: async (client: Client) => { + return client.end() + }, + validate: (client: Client) => { + return Promise.resolve(!client.closed); + } +}, { + testOnBorrow: true, + max: 10 +}) + +const executeQuery = (query: Query) => { + console.log('executing query'); + + const res = pool.acquire(); + + return res.then(async (client: Client) => { + console.log('aquired client from pool') + const result = await client.execute(query); + pool.release(client).then(() => { + console.log("client released back into pool") + }); + return result; + }).catch((err: any) => { + console.log(err); + }) +} + +export const shutdown = () => { + pool.drain() + .then(() => { + return pool.clear(); + }); +} + +export const insert = (table: string, params: any) => { + + let fileName = params.fileName; + let folderName = params.folderName; + let fusionID = null; + let username = null; + let size = null; + let version = null; + + if (params.hasOwnProperty('fusionID')) {fusionID = params.fusionID} + if (params.hasOwnProperty('username')) {username = params.username} + if (params.hasOwnProperty('size')) {size = params.size} + if (params.hasOwnProperty('version')) {version = params.version} + + let query: Query; + + console.log("params: ", params); + + switch(table) { + case 'fusion': + query = new Query ( + "INSERT INTO fusion (fusion_id, folder_name, file_name, username, size, version)" + + "VALUES ($1, $2, $3, $4, $5, $6)", + [fusionID, folderName, fileName, username, size, version] + ); + break; + case 'local': + query = new Query ( + "INSERT INTO local (file_name, folder_name, fusion_id)" + + "VALUES ($1, $2, $3)", + [fileName, folderName, fusionID] + ); + break; + case 'archive': + query = new Query ( + "INSERT INTO archive (fusion_id, folder_name, file_name, username, size, version)" + + "VALUES ($1, $2, $3, $4, $5, $6)", + [fusionID, folderName, fileName, username, size, version] + ); + break; + default: + throw new Error("Invalid table/params"); + } + + return executeQuery(query); +} \ No newline at end of file diff --git a/src/old/taskRunner.ts b/src/old/taskRunner.ts index 53cb8b5..03643d4 100644 --- a/src/old/taskRunner.ts +++ b/src/old/taskRunner.ts @@ -3,9 +3,9 @@ const watcher = require('./watcher'); const downloader = require('./downloader'); const fs = require('fs'); import { AuthToken } from 'forge-apis'; -import * as config from './config'; -import * as localhook from './localhook'; -import * as uploader from './uploadfile'; +import * as config from '../config'; +import * as localhook from '../localhook'; +import * as uploader from '../uploadfile'; /** diff --git a/src/old/watcher.ts b/src/old/watcher.ts index e07664b..24f37c5 100644 --- a/src/old/watcher.ts +++ b/src/old/watcher.ts @@ -1,4 +1,4 @@ -// const axios = require('axios'); +const axios = require('axios'); /** * Kevin Pan | pan261@purdue.edu | Last Modified: 12/2/2021 diff --git a/src/server.ts b/src/server.ts index 3d726fb..1daad82 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2,6 +2,7 @@ import express = require('express'); import bodyParser = require('body-parser'); import crypto = require('crypto'); import * as config from './config'; +import * as db from './db'; import fs = require('fs'); const webhooks = require('./webhooks'); @@ -16,7 +17,7 @@ const downloader = require('./downloader'); * - FORGE_CLIENT_ID * - FORGE CLIENT_SECRET * - FORGE_CALLBACK_URL - * - PORT (defaults to 3000) + * - PORT_SERVER (defaults to 3000) * * These are all configured in config.js * @@ -143,19 +144,28 @@ const createServer = () => { const itemID = body.payload.lineageUrn; //TODO: check if extension is in the name const fileName = body.payload.name; - const destination = __dirname + '/FileShare' + config.folderIDtoLocal[body.payload.parentFolderUrn]; + const destination = __dirname + '/file_share' + config.folderIDtoLocal[body.payload.parentFolderUrn]; downloader.download(config.projectID, itemID, fileName, destination, credentials); } }); + // Will make some server endpoints to test db + app.get('/db/insert', async function (req: any, res: any) { + const result = await db.insert("local", { + fileName: crypto.randomBytes(10).toString('hex'), + folderName: 'Mill', + }); + res.send(result); + }); + // Default endpoint app.use((err: any, req: any, res: any, next: any) => { console.error(err); res.status(err.statusCode).json(err); }); - const server = app.listen(config.port, () => { console.log(`Server listening on port ${PORT}`); }); + const server = app.listen(PORT, () => { console.log(`Server listening on port ${PORT}`); }); return server; } @@ -181,4 +191,16 @@ const refresh = () => { }); }; -createServer(); \ No newline at end of file +const server = createServer(); + +/** + * May eventually need this to drain db connection pool + * With docker, ctrl+c doesn't work (apparently it is not a SIGINT), + * need to open a new terminal and run `docker-compose stop` + */ +process.on('SIGTERM', async () => { + console.log("shutting down node process") + await db.shutdown(); + if (server) {server.close()} + process.exit(0) +}); \ No newline at end of file diff --git a/src/webhooks.ts b/src/webhooks.ts index b847c82..a070ae7 100644 --- a/src/webhooks.ts +++ b/src/webhooks.ts @@ -16,7 +16,7 @@ import * as config from './config'; * */ -const FileShareID = "urn:adsk.wipprod:fs.folder:co.T0n0mYQeS16K0lq1VuuYVQ" +const fileShareID = "urn:adsk.wipprod:fs.folder:co.T0n0mYQeS16K0lq1VuuYVQ" /** * Function to manage hook lifecycle @@ -78,7 +78,7 @@ const createHook = (event: string, credentials: any) => { autoReactivateHook: "true", callbackUrl: config.hookCallbackURL, scope: { - folder: FileShareID + folder: fileShareID } } }).then((res: any) => {