postgres client pooling + testing insert in server

docker-compose.yml

@@ -2,10 +2,23 @@ version: "3.8"
 
 services:
 
+  # access through `psql -h localhost -p 5432 -d filesync -U user`, password is 'pass'
+  # to be able to build after the initializing the db, you may need to run `sudo chown -R $USER db_data/`
+  database:
+    image: 'postgres:13'
+    env_file: ./.env
+    volumes:
+      - ./postgres/db_data/:/var/lib/postgresql/data/
+      - ./postgres/sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
+    ports:
+      - ${PORT_DB}:${PORT_DB}
+
   server:
     build:
       context: .
       dockerfile: Dockerfile
+    depends_on:
+      - database
     env_file: ./.env
     volumes: 
       - ./src/build:/ffs/
@@ -17,15 +30,4 @@ services:
       - ${LOCAL_DIR_WATERJET}:/ffs/file_share/Waterjet/
     ports:
       - ${PORT_SERVER}:${PORT_SERVER}
-    command: "node server.js"
-
-  # access through `psql -h localhost -p 5432 -d filesync -U user`, password is 'pass'
-  # to be able to build after the initializing the db, you may need to run `sudo chown -R $USER db_data/`
-  database:
-    image: 'postgres:13'
-    env_file: ./.env
-    volumes:
-      - ./postgres/db_data/:/var/lib/postgresql/data/
-      - ./postgres/sql/create_tables.sql:/docker-entrypoint-initdb.d/create_tables.sql
-    ports:
-      - ${PORT_DB}:${PORT_DB}
+    command: "node server.js"

environment.d.ts

@@ -4,7 +4,8 @@ declare global {
         FORGE_CLIENT_ID: string;
         FORGE_CLIENT_SECRET: string;
         FORGE_CALLBACK_URL: string;
-        PORT_SERVER: string;
+        PORT_SERVER: number;
+        PORT_DB: number;
         WEBHOOK_TOKEN: string;
       }
     }

package.json

@@ -24,7 +24,9 @@
     "crypto": "^1.0.1",
     "dotenv": "^10.0.0",
     "express": "^4.17.1",
-    "forge-apis": "^0.8.6"
+    "forge-apis": "^0.8.6",
+    "generic-pool": "^3.8.2",
+    "ts-postgres": "^1.2.1"
   },
   "devDependencies": {
     "@types/dotenv": "^8.2.0",

postgres/sql/create_tables.sql

@@ -1,23 +1,29 @@
+-- Primary keys must include folder name in order to avoid collisions
+-- between filenames in different folders
+
 CREATE TABLE IF NOT EXISTS fusion (
   fusion_id varchar(250) NOT NULL,
+  folder_name varchar(250) NOT NULL,
   file_name varchar(250) NOT NULL,
-  username varchar(250) NOT NULL,
-  size int NOT NULL,
-  version int NOT NULL,
+  username varchar(250),
+  size int,
+  version int,
   PRIMARY KEY (fusion_id)
 );
 
 CREATE TABLE IF NOT EXISTS local (
   file_name varchar(250) NOT NULL,
-  fusion_id varchar(250) NOT NULL,
-  PRIMARY KEY (file_name)
+  folder_name varchar(250) NOT NULL,
+  fusion_id varchar(250),
+  PRIMARY KEY (file_name, folder_name)
 );
 
 CREATE TABLE IF NOT EXISTS archive (
   fusion_id varchar(250) NOT NULL,
   file_name varchar(250) NOT NULL,
+  folder_name varchar(250) NOT NULL,
   username varchar(250) NOT NULL,
-  size int NOT NULL,
-  version int NOT NULL,
-  PRIMARY KEY (fusion_id)
+  size int,
+  version int,
+  PRIMARY KEY (fusion_id, folder_name)
 );

src/config.ts

@@ -3,9 +3,7 @@
  */
 
 import { AuthClientThreeLegged } from 'forge-apis';
-
 import * as dotenv from 'dotenv';
-
 dotenv.config({path: '../.env'});
 
 export const ForgeAuthClient = AuthClientThreeLegged;
@@ -58,4 +56,8 @@ export const forgeClientId = process.env.FORGE_CLIENT_ID;
 export const forgeClientSecret = process.env.FORGE_CLIENT_SECRET;
 export const forgeCallbackURL = process.env.FORGE_CALLBACK_URL;
 export const hookCallbackURL = `${process.env.HOOK_CALLBACK_HOSTNAME}/hook`;
-export const webhookToken = process.env.WEBHOOK_TOKEN;
+export const webhookToken = process.env.WEBHOOK_TOKEN;
+export const postgresUser = process.env.POSTGRES_USER;
+export const postgresPassword = process.env.POSTGRES_PASSWORD;
+export const postgresPort = process.env.PORT_DB;
+export const postgresDB = process.env.POSTGRES_DB;

src/db.ts

@@ -0,0 +1,108 @@
+import { Client, Query } from 'ts-postgres';
+import { createPool } from 'generic-pool';
+import * as config from './config';
+
+const pool = createPool({
+  create: async () => {
+    return new Promise<Client>((resolve, reject) => {
+
+      const client = new Client({
+        'host': 'database', // name of docker container, not 'localhost'
+        'port': config.postgresPort,
+        'user': config.postgresUser,
+        'password': config.postgresPassword,
+        'database': config.postgresDB
+      });
+
+      client.connect()
+      .then(() => {
+        console.log("successfully connected to db")
+        resolve(client);
+      }).catch((err) => {
+        console.log()
+        console.log(err)
+        reject(err);
+      });
+    });
+  },
+  destroy: async (client: Client) => {
+    return client.end()
+  },
+  validate: (client: Client) => {
+    return Promise.resolve(!client.closed);
+  }
+}, {
+  testOnBorrow: true,
+  max: 10
+})
+
+const executeQuery = (query: Query) => {
+  console.log('executing query');
+
+  const res = pool.acquire();
+
+  return res.then(async (client: Client) => {
+    console.log('aquired client from pool')
+    const result = await client.execute(query);
+    pool.release(client).then(() => {
+      console.log("client released back into pool")
+    });
+    return result;
+  }).catch(err => {
+    console.log(err);
+  })
+}
+
+export const shutdown = () => {
+  pool.drain()
+  .then(() => {
+    return pool.clear();
+  });
+}
+
+export const insert = (table: string, params: any) => {
+
+  let fileName = params.fileName;
+  let folderName = params.folderName;
+  let fusionID = null;
+  let username = null;
+  let size = null;
+  let version = null;
+
+  if (params.hasOwnProperty('fusionID')) {fusionID = params.fusionID}
+  if (params.hasOwnProperty('username')) {username = params.username}
+  if (params.hasOwnProperty('size')) {size = params.size}
+  if (params.hasOwnProperty('version')) {version = params.version}
+
+  let query: Query;
+
+  console.log("params: ", params);
+
+  switch(table) {
+    case 'fusion':
+      query = new Query (
+        "INSERT INTO fusion (fusion_id, folder_name, file_name, username, size, version)" +
+        "VALUES ($1, $2, $3, $4, $5, $6)",
+        [fusionID, folderName, fileName, username, size, version]
+      );
+      break;
+    case 'local':
+      query = new Query (
+        "INSERT INTO local (file_name, folder_name, fusion_id)" +
+        "VALUES ($1, $2, $3)",
+        [fileName, folderName, fusionID]
+      );
+      break;
+    case 'archive':
+      query = new Query (
+        "INSERT INTO archive (fusion_id, folder_name, file_name, username, size, version)" +
+        "VALUES ($1, $2, $3, $4, $5, $6)",
+        [fusionID, folderName, fileName, username, size, version]
+      );
+      break;
+    default:
+      throw new Error("Invalid table/params");
+  }
+
+  return executeQuery(query);
+}

src/old/watcher.ts

@@ -1,4 +1,4 @@
-// const axios = require('axios');
+const axios = require('axios');
 
 /**
  * Kevin Pan | pan261@purdue.edu | Last Modified: 12/2/2021

src/server.ts

@@ -2,6 +2,7 @@ import express = require('express');
 import bodyParser = require('body-parser');
 import crypto = require('crypto');
 import * as config from './config';
+import * as db from './db';
 import fs = require('fs');
 
 const webhooks = require('./webhooks');
@@ -149,6 +150,15 @@ const createServer = () => {
     }
   });
 
+  // Will make some server endpoints to test db
+  app.get('/db/insert', async function (req: any, res: any) {
+    const result = await db.insert("local", {
+      fileName: crypto.randomBytes(10).toString('hex'),
+      folderName: 'Mill',
+    });
+    res.send(result);
+  });
+
   // Default endpoint
   app.use((err: any, req: any, res: any, next: any) => {
     console.error(err);
@@ -181,4 +191,16 @@ const refresh = () => {
     });
 };
 
-createServer();
+const server = createServer();
+
+/**
+ * May eventually need this to drain db connection pool
+ * With docker, ctrl+c doesn't work (apparently it is not a SIGINT),
+ * need to open a new terminal and run `docker-compose stop`
+ */
+process.on('SIGTERM',  async () => {
+  console.log("shutting down node process")
+  await db.shutdown();
+  if (server) {server.close()}
+  process.exit(0)
+});