diff --git a/docker-compose.yml b/docker-compose.yml index 0d555c4..8b97cc7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,10 +2,23 @@ version: "3.8" services: + # access through `psql -h localhost -p 5432 -d filesync -U user`, password is 'pass' + # to be able to build after the initializing the db, you may need to run `sudo chown -R $USER db_data/` + database: + image: 'postgres:13' + env_file: ./.env + volumes: + - ./postgres/db_data/:/var/lib/postgresql/data/ + - ./postgres/sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql + ports: + - ${PORT_DB}:${PORT_DB} + server: build: context: . dockerfile: Dockerfile + depends_on: + - database env_file: ./.env volumes: - ./src/build:/ffs/ @@ -17,15 +30,4 @@ services: - ${LOCAL_DIR_WATERJET}:/ffs/file_share/Waterjet/ ports: - ${PORT_SERVER}:${PORT_SERVER} - command: "node server.js" - - # access through `psql -h localhost -p 5432 -d filesync -U user`, password is 'pass' - # to be able to build after the initializing the db, you may need to run `sudo chown -R $USER db_data/` - database: - image: 'postgres:13' - env_file: ./.env - volumes: - - ./postgres/db_data/:/var/lib/postgresql/data/ - - ./postgres/sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql - ports: - - ${PORT_DB}:${PORT_DB} \ No newline at end of file + command: "node server.js" \ No newline at end of file diff --git a/environment.d.ts b/environment.d.ts index 6dd4467..e91e6ca 100644 --- a/environment.d.ts +++ b/environment.d.ts @@ -4,7 +4,8 @@ declare global { FORGE_CLIENT_ID: string; FORGE_CLIENT_SECRET: string; FORGE_CALLBACK_URL: string; - PORT_SERVER: string; + PORT_SERVER: number; + PORT_DB: number; WEBHOOK_TOKEN: string; } } diff --git a/package-lock.json b/package-lock.json index e0c5115..170eb12 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,7 +15,9 @@ "crypto": "^1.0.1", "dotenv": "^10.0.0", "express": "^4.17.1", - "forge-apis": "^0.8.6" + "forge-apis": "^0.8.6", + "generic-pool": "^3.8.2", + "ts-postgres": "^1.2.1" }, "devDependencies": { "@types/dotenv": "^8.2.0", @@ -1632,6 +1634,14 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "node_modules/generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==", + "engines": { + "node": ">= 4" + } + }, "node_modules/get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -3094,6 +3104,22 @@ "node": ">=0.8" } }, + "node_modules/ts-postgres": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ts-postgres/-/ts-postgres-1.2.1.tgz", + "integrity": "sha512-5cCIdGMMlX8g+cFs0DFykLKyTmPvJ9gCv5TR13R90Azz8gzEtugYq6gKm5glBXF66gtvsP/lUashU5akfQsFaA==", + "dependencies": { + "ts-typed-events": "^2.0.1" + }, + "engines": { + "node": ">=10.7.0" + } + }, + "node_modules/ts-typed-events": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ts-typed-events/-/ts-typed-events-2.0.1.tgz", + "integrity": "sha512-7V+vCn52mZVSe+udf0L2SRkvs8BquF7xOtVktJsiyz3IGgcAhHwp3aetPcrxT5Xq23Bb/tBX0azQ6BBvPVytZg==" + }, "node_modules/tsconfig-paths": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", @@ -4566,6 +4592,11 @@ "integrity": "sha1-GwqzvVU7Kg1jmdKcDj6gslIHgyc=", "dev": true }, + "generic-pool": { + "version": "3.8.2", + "resolved": "https://registry.npmjs.org/generic-pool/-/generic-pool-3.8.2.tgz", + "integrity": "sha512-nGToKy6p3PAbYQ7p1UlWl6vSPwfwU6TMSWK7TTu+WUY4ZjyZQGniGGt2oNVvyNSpyZYSB43zMXVLcBm08MTMkg==" + }, "get-intrinsic": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", @@ -5645,6 +5676,19 @@ "punycode": "^2.1.1" } }, + "ts-postgres": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/ts-postgres/-/ts-postgres-1.2.1.tgz", + "integrity": "sha512-5cCIdGMMlX8g+cFs0DFykLKyTmPvJ9gCv5TR13R90Azz8gzEtugYq6gKm5glBXF66gtvsP/lUashU5akfQsFaA==", + "requires": { + "ts-typed-events": "^2.0.1" + } + }, + "ts-typed-events": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/ts-typed-events/-/ts-typed-events-2.0.1.tgz", + "integrity": "sha512-7V+vCn52mZVSe+udf0L2SRkvs8BquF7xOtVktJsiyz3IGgcAhHwp3aetPcrxT5Xq23Bb/tBX0azQ6BBvPVytZg==" + }, "tsconfig-paths": { "version": "3.12.0", "resolved": "https://registry.npmjs.org/tsconfig-paths/-/tsconfig-paths-3.12.0.tgz", diff --git a/package.json b/package.json index fb712d2..e40f32d 100644 --- a/package.json +++ b/package.json @@ -24,7 +24,9 @@ "crypto": "^1.0.1", "dotenv": "^10.0.0", "express": "^4.17.1", - "forge-apis": "^0.8.6" + "forge-apis": "^0.8.6", + "generic-pool": "^3.8.2", + "ts-postgres": "^1.2.1" }, "devDependencies": { "@types/dotenv": "^8.2.0", diff --git a/postgres/sql/create_tables.sql b/postgres/sql/create_tables.sql index f5ea731..2e09be1 100644 --- a/postgres/sql/create_tables.sql +++ b/postgres/sql/create_tables.sql @@ -1,23 +1,29 @@ +-- Primary keys must include folder name in order to avoid collisions +-- between filenames in different folders + CREATE TABLE IF NOT EXISTS fusion ( fusion_id varchar(250) NOT NULL, + folder_name varchar(250) NOT NULL, file_name varchar(250) NOT NULL, - username varchar(250) NOT NULL, - size int NOT NULL, - version int NOT NULL, + username varchar(250), + size int, + version int, PRIMARY KEY (fusion_id) ); CREATE TABLE IF NOT EXISTS local ( file_name varchar(250) NOT NULL, - fusion_id varchar(250) NOT NULL, - PRIMARY KEY (file_name) + folder_name varchar(250) NOT NULL, + fusion_id varchar(250), + PRIMARY KEY (file_name, folder_name) ); CREATE TABLE IF NOT EXISTS archive ( fusion_id varchar(250) NOT NULL, file_name varchar(250) NOT NULL, + folder_name varchar(250) NOT NULL, username varchar(250) NOT NULL, - size int NOT NULL, - version int NOT NULL, - PRIMARY KEY (fusion_id) + size int, + version int, + PRIMARY KEY (fusion_id, folder_name) ); \ No newline at end of file diff --git a/src/config.ts b/src/config.ts index fc0654f..91b6855 100644 --- a/src/config.ts +++ b/src/config.ts @@ -3,9 +3,7 @@ */ import { AuthClientThreeLegged } from 'forge-apis'; - import * as dotenv from 'dotenv'; - dotenv.config({path: '../.env'}); export const ForgeAuthClient = AuthClientThreeLegged; @@ -58,4 +56,8 @@ export const forgeClientId = process.env.FORGE_CLIENT_ID; export const forgeClientSecret = process.env.FORGE_CLIENT_SECRET; export const forgeCallbackURL = process.env.FORGE_CALLBACK_URL; export const hookCallbackURL = `${process.env.HOOK_CALLBACK_HOSTNAME}/hook`; -export const webhookToken = process.env.WEBHOOK_TOKEN; \ No newline at end of file +export const webhookToken = process.env.WEBHOOK_TOKEN; +export const postgresUser = process.env.POSTGRES_USER; +export const postgresPassword = process.env.POSTGRES_PASSWORD; +export const postgresPort = process.env.PORT_DB; +export const postgresDB = process.env.POSTGRES_DB; \ No newline at end of file diff --git a/src/db.ts b/src/db.ts new file mode 100644 index 0000000..5e09b99 --- /dev/null +++ b/src/db.ts @@ -0,0 +1,108 @@ +import { Client, Query } from 'ts-postgres'; +import { createPool } from 'generic-pool'; +import * as config from './config'; + +const pool = createPool({ + create: async () => { + return new Promise((resolve, reject) => { + + const client = new Client({ + 'host': 'database', // name of docker container, not 'localhost' + 'port': config.postgresPort, + 'user': config.postgresUser, + 'password': config.postgresPassword, + 'database': config.postgresDB + }); + + client.connect() + .then(() => { + console.log("successfully connected to db") + resolve(client); + }).catch((err) => { + console.log() + console.log(err) + reject(err); + }); + }); + }, + destroy: async (client: Client) => { + return client.end() + }, + validate: (client: Client) => { + return Promise.resolve(!client.closed); + } +}, { + testOnBorrow: true, + max: 10 +}) + +const executeQuery = (query: Query) => { + console.log('executing query'); + + const res = pool.acquire(); + + return res.then(async (client: Client) => { + console.log('aquired client from pool') + const result = await client.execute(query); + pool.release(client).then(() => { + console.log("client released back into pool") + }); + return result; + }).catch(err => { + console.log(err); + }) +} + +export const shutdown = () => { + pool.drain() + .then(() => { + return pool.clear(); + }); +} + +export const insert = (table: string, params: any) => { + + let fileName = params.fileName; + let folderName = params.folderName; + let fusionID = null; + let username = null; + let size = null; + let version = null; + + if (params.hasOwnProperty('fusionID')) {fusionID = params.fusionID} + if (params.hasOwnProperty('username')) {username = params.username} + if (params.hasOwnProperty('size')) {size = params.size} + if (params.hasOwnProperty('version')) {version = params.version} + + let query: Query; + + console.log("params: ", params); + + switch(table) { + case 'fusion': + query = new Query ( + "INSERT INTO fusion (fusion_id, folder_name, file_name, username, size, version)" + + "VALUES ($1, $2, $3, $4, $5, $6)", + [fusionID, folderName, fileName, username, size, version] + ); + break; + case 'local': + query = new Query ( + "INSERT INTO local (file_name, folder_name, fusion_id)" + + "VALUES ($1, $2, $3)", + [fileName, folderName, fusionID] + ); + break; + case 'archive': + query = new Query ( + "INSERT INTO archive (fusion_id, folder_name, file_name, username, size, version)" + + "VALUES ($1, $2, $3, $4, $5, $6)", + [fusionID, folderName, fileName, username, size, version] + ); + break; + default: + throw new Error("Invalid table/params"); + } + + return executeQuery(query); +} \ No newline at end of file diff --git a/src/old/watcher.ts b/src/old/watcher.ts index 1e9ed3a..4824607 100644 --- a/src/old/watcher.ts +++ b/src/old/watcher.ts @@ -1,4 +1,4 @@ -// const axios = require('axios'); +const axios = require('axios'); /** * Kevin Pan | pan261@purdue.edu | Last Modified: 12/2/2021 diff --git a/src/server.ts b/src/server.ts index 213ad64..1daad82 100644 --- a/src/server.ts +++ b/src/server.ts @@ -2,6 +2,7 @@ import express = require('express'); import bodyParser = require('body-parser'); import crypto = require('crypto'); import * as config from './config'; +import * as db from './db'; import fs = require('fs'); const webhooks = require('./webhooks'); @@ -149,6 +150,15 @@ const createServer = () => { } }); + // Will make some server endpoints to test db + app.get('/db/insert', async function (req: any, res: any) { + const result = await db.insert("local", { + fileName: crypto.randomBytes(10).toString('hex'), + folderName: 'Mill', + }); + res.send(result); + }); + // Default endpoint app.use((err: any, req: any, res: any, next: any) => { console.error(err); @@ -181,4 +191,16 @@ const refresh = () => { }); }; -createServer(); \ No newline at end of file +const server = createServer(); + +/** + * May eventually need this to drain db connection pool + * With docker, ctrl+c doesn't work (apparently it is not a SIGINT), + * need to open a new terminal and run `docker-compose stop` + */ +process.on('SIGTERM', async () => { + console.log("shutting down node process") + await db.shutdown(); + if (server) {server.close()} + process.exit(0) +}); \ No newline at end of file