Merge remote-tracking branch origin/main into mb/revert-transactions-…

…for-workers
HHS · Jul 19, 2024 · 6b53dbe · 6b53dbe
2 parents f749515 + a3ff0df
commit 6b53dbe
Show file tree

Hide file tree

Showing 16 changed files with 262 additions and 208 deletions.
diff --git a/.cfignore b/.cfignore
@@ -12,3 +12,5 @@ hses.zip
 temp/
 tests/
 .tmp/
+*.test.js
+!*.sql
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -406,7 +406,7 @@ parameters:
     default: "al-ttahub-2939-add-fei-root-cause-to-goal-card"
     type: string
   sandbox_git_branch:  # change to feature branch to test deployment
-    default: "jp/3112/rm-elasticsearch"
+    default: "gh/cfignore-keep-sql-drop-tests"
     type: string
   prod_new_relic_app_id:
     default: "877570491"

diff --git a/frontend/package.json b/frontend/package.json
@@ -187,9 +187,13 @@
     "jest-fetch-mock": "^3.0.3",
     "jest-junit": "^13.0.0",
     "mutationobserver-shim": "^0.3.7",
+    "postcss": "^8.4.39",
     "react-scripts": "^5.0.1",
     "react-select-event": "^5.1.0"
   },
+  "peerDependencies": {
+    "postcss": "^8.4.33"
+  },
   "jest": {
     "coveragePathIgnorePatterns": [
       "<rootDir>/src/index.js",

diff --git a/frontend/public/index.html b/frontend/public/index.html
@@ -3,8 +3,6 @@
   <head>
     <meta charset="utf-8" />
     <link rel="icon" href="%PUBLIC_URL%/logo64.png" />
-    <link rel="preconnect" href="https://fonts.gstatic.com">
-    <link href="https://fonts.googleapis.com/css2?family=Merriweather&display=swap" rel="stylesheet">
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <meta name="theme-color" content="#000000" />
     <meta

diff --git a/frontend/src/App.scss b/frontend/src/App.scss
@@ -9,6 +9,34 @@
   font-weight: bold;
 }
 
+@font-face {
+  font-family: 'Merriweather';
+  src: url('./assets/Merriweather-Regular.ttf') format('truetype');
+  font-weight: normal;
+}
+
+@font-face {
+  font-family: 'Merriweather';
+  src: url('./assets/Merriweather-Bold.ttf') format('truetype');
+  font-weight: bold;
+}
+
+@font-face {
+  font-family: 'Merriweather';
+  src: url('./assets/Merriweather-Italic.ttf') format('truetype');
+  font-style: italic;
+  font-weight: normal;
+}
+
+@font-face {
+  font-family: 'Merriweather';
+  src: url('./assets/Merriweather-BoldItalic.ttf') format('truetype');
+  font-style: italic;
+  font-weight: bold;
+}
+
+
+
 a {
   color: $text-link;
 }

diff --git a/frontend/src/assets/Merriweather-Bold.ttf b/frontend/src/assets/Merriweather-Bold.ttf
diff --git a/frontend/src/assets/Merriweather-BoldItalic.ttf b/frontend/src/assets/Merriweather-BoldItalic.ttf
diff --git a/frontend/src/assets/Merriweather-Italic.ttf b/frontend/src/assets/Merriweather-Italic.ttf
diff --git a/frontend/src/assets/Merriweather-Regular.ttf b/frontend/src/assets/Merriweather-Regular.ttf
diff --git a/frontend/yarn.lock b/frontend/yarn.lock
@@ -8484,7 +8484,7 @@ mutationobserver-shim@^0.3.7:
   resolved "https://registry.yarnpkg.com/mutationobserver-shim/-/mutationobserver-shim-0.3.7.tgz#8bf633b0c0b0291a1107255ed32c13088a8c5bf3"
   integrity sha512-oRIDTyZQU96nAiz2AQyngwx1e89iApl2hN5AOYwyxLUB47UYsU3Wv9lJWqH5y/QdiYkc5HQLi23ZNB3fELdHcQ==
 
-nanoid@^3.2.0, nanoid@^3.3.6:
+nanoid@^3.2.0, nanoid@^3.3.6, nanoid@^3.3.7:
   version "3.3.4"
   resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab"
   integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==
@@ -8964,6 +8964,11 @@ picocolors@^1.0.0:
   resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c"
   integrity sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==
 
+picocolors@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.1.tgz#a8ad579b571952f0e5d25892de5445bcfe25aaa1"
+  integrity sha512-anP1Z8qwhkbmu7MFP5iTt+wQKXgwzf7zTyGlcdzabySa9vd0Xt392U0rVmz9poOaBj0uHJKyyo9/upk0HrEQew==
+
 picomatch@^2.0.4, picomatch@^2.2.1, picomatch@^2.2.2, picomatch@^2.2.3, picomatch@^2.3.1:
   version "2.3.1"
   resolved "https://registry.yarnpkg.com/picomatch/-/picomatch-2.3.1.tgz#3ba3833733646d9d3e4995946c1365a67fb07a42"
@@ -9612,6 +9617,15 @@ postcss@^7.0.35, postcss@^8.3.5, postcss@^8.4.18, postcss@^8.4.19, postcss@^8.4.
     picocolors "^1.0.0"
     source-map-js "^1.0.2"
 
+postcss@^8.4.39:
+  version "8.4.39"
+  resolved "https://registry.yarnpkg.com/postcss/-/postcss-8.4.39.tgz#aa3c94998b61d3a9c259efa51db4b392e1bde0e3"
+  integrity sha512-0vzE+lAiG7hZl1/9I8yzKLx3aR9Xbof3fBHKunvMfOCYAtMhrsnccJY2iTURb9EZd5+pLuiNV9/c/GZJOHsgIw==
+  dependencies:
+    nanoid "^3.3.7"
+    picocolors "^1.0.1"
+    source-map-js "^1.2.0"
+
 potpack@^1.0.1:
   version "1.0.2"
   resolved "https://registry.yarnpkg.com/potpack/-/potpack-1.0.2.tgz#23b99e64eb74f5741ffe7656b5b5c4ddce8dfc14"
@@ -10971,6 +10985,11 @@ source-list-map@^2.0.0, source-list-map@^2.0.1:
   resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.0.2.tgz#adbc361d9c62df380125e7f161f71c826f1e490c"
   integrity sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==
 
+source-map-js@^1.2.0:
+  version "1.2.0"
+  resolved "https://registry.yarnpkg.com/source-map-js/-/source-map-js-1.2.0.tgz#16b809c162517b5b8c3e7dcd315a2a5c2612b2af"
+  integrity sha512-itJW8lvSA0TXEphiRoawsCksnlf8SyvmFzIhltqAHluXd88pkCd+cXJVHTDwdCr0IzwptSm035IHQktUu1QUMg==
+
 source-map-loader@^3.0.0:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/source-map-loader/-/source-map-loader-3.0.2.tgz#af23192f9b344daa729f6772933194cc5fa54fee"

diff --git a/src/app.js b/src/app.js
@@ -42,6 +42,7 @@ process.on('unhandledRejection', (reason, promise) => {
 });
 
 const app = express();
+
 const oauth2CallbackPath = '/oauth2-client/login/oauth2/code/';
 let index;
 
@@ -60,10 +61,27 @@ app.use(express.json({ limit: '2MB' }));
 app.use(express.urlencoded({ extended: true }));
 
 app.use((req, res, next) => {
+  // set the X-Content-Type-Options header to prevent MIME-sniffing
+  res.set('X-Content-Type-Options', 'nosniff');
+
+  // set nonce
   res.locals.nonce = crypto.randomBytes(16).toString('hex');
+
+  // set CSP
   const cspMiddleware = helmet.contentSecurityPolicy({
     directives: {
-      ...omit(helmet.contentSecurityPolicy.getDefaultDirectives(), 'upgrade-insecure-requests', 'block-all-mixed-content', 'script-src', 'img-src', 'default-src'),
+      ...omit(
+        helmet.contentSecurityPolicy.getDefaultDirectives(),
+        'upgrade-insecure-requests',
+        'block-all-mixed-content',
+        'script-src',
+        'img-src',
+        'default-src',
+        'style-src',
+        'font-src',
+      ),
+      styleSrc: ["'self'", "'unsafe-inline'"],
+      fontSrc: ["'self'"],
       'form-action': ["'self'"],
       scriptSrc: ["'self'", '*.googletagmanager.com'],
       scriptSrcElem: ["'self'", 'https://*.googletagmanager.com', `'nonce-${res.locals.nonce}'`],
@@ -81,9 +99,11 @@ if (process.env.NODE_ENV === 'production' || process.env.NODE_ENV === 'dss') {
 }
 
 app.use('/api/v1', require('./routes/externalApi').default);
-
 app.use('/api', require('./routes/apiDirectory').default);
 
+// Disable "X-Powered-By" header
+app.disable('x-powered-by');
+
 // TODO: change `app.get...` with `router.get...` once our oauth callback has been updated
 app.get(oauth2CallbackPath, cookieSession, async (req, res) => {
   try {

diff --git a/src/lib/importSystem/process.ts b/src/lib/importSystem/process.ts
@@ -487,9 +487,10 @@ const processZipFileFromS3 = async (
 
   // Destructure properties from the importFile object
   const {
-    dataValues: { importFileId, processAttempts = 0 },
-    file: { key },
-    import: { definitions: processDefinitions },
+    importFileId,
+    processAttempts = 0,
+    fileKey: key,
+    importDefinitions: processDefinitions,
   } = importFile;
 
   // These must be let to properly wrap the population in a try/catch

diff --git a/src/lib/importSystem/record.ts b/src/lib/importSystem/record.ts
@@ -1,13 +1,9 @@
 import {
   Sequelize,
   fn,
-  cast,
-  col,
   literal,
   Op,
-  where,
 } from 'sequelize';
-import { Readable } from 'stream';
 import { v4 as uuidv4 } from 'uuid';
 import { FileInfo as FTPFileInfo, FileListing } from '../stream/sftp';
 import { SchemaNode } from '../stream/xml';
@@ -197,32 +193,18 @@ const getNextFileToProcess = async (
     },
   );
 
-  // Find the next import file to process
+  // Find the next import file to process without join and locking mechanism
   const importFile = await ImportFile.findOne({
     attributes: [
-      ['id', 'importFileId'],
+      'id',
       'fileId',
       'status',
       'processAttempts',
-    ],
-    include: [
-      {
-        model: File,
-        as: 'file',
-        attributes: [
-          'key',
-        ],
-      },
-      {
-        model: Import,
-        as: 'import',
-        attributes: [
-          'definitions',
-        ],
-      },
+      'importId',
     ],
     where: {
       importId,
+      fileId: { [Op.ne]: null }, // Ensure fileId is not null
       [Op.or]: [
         // New Work
         { status: IMPORT_STATUSES.COLLECTED }, // Import file is in the "collected" status
@@ -240,9 +222,38 @@ const getNextFileToProcess = async (
     ],
     limit: 1, // Limit the result to 1 record
     lock: true, // Lock the row for update to prevent race conditions
+    raw: true,
   });
 
-  return importFile;
+  if (!importFile) {
+    return null;
+  }
+
+  // Fetch the associated File data
+  const file = await File.findOne({
+    attributes: ['key'],
+    where: {
+      id: importFile.fileId,
+    },
+    raw: true,
+  });
+
+  // Fetch the associated Import data
+  const importData = await Import.findOne({
+    attributes: ['definitions'],
+    where: {
+      id: importFile.importId,
+    },
+    raw: true,
+  });
+  return {
+    importFileId: importFile.id,
+    fileId: importFile.fileId,
+    status: importFile.status,
+    processAttempts: importFile.processAttempts,
+    fileKey: file?.key,
+    importDefinitions: importData?.definitions,
+  };
 };
 
 /**
@@ -381,11 +392,10 @@ const recordAvailableDataFiles = async (
   });
 
   const fileMatches = (currentImportDataFile, availableFile) => (
-    importFileId === currentImportDataFile.importFileId
+    importFileId === currentImportDataFile?.importFileId
     && availableFile.path === currentImportDataFile.fileInfo.path
     && availableFile.name === currentImportDataFile.fileInfo.name
   );
-
   // Separate the available files into new, matched, and removed files
   // New files are those that are not already recorded in the database
   const newFiles = availableFiles
@@ -508,13 +518,10 @@ const logFileToBeCollected = async (
 }> => {
   let key;
 
-  // Find the import file record based on the import ID and available file information
+  // Step 1: Find and lock the import file record based on the import ID and available
+  // file information
   const importFile = await ImportFile.findOne({
-    attributes: [
-      'id',
-      'fileId',
-      'downloadAttempts',
-    ],
+    attributes: ['id', 'fileId', 'downloadAttempts'],
     where: {
       importId,
       ftpFileInfo: {
@@ -524,15 +531,16 @@ const logFileToBeCollected = async (
         },
       },
     },
-    include: [{
-      model: File,
-      as: 'file',
-      attributes: ['key'],
-      require: false,
-    }],
     lock: true, // Lock the row for update to prevent race conditions
+    raw: true,
   });
 
+  if (!importFile) {
+    throw new Error('Import file not found');
+  }
+
+  const downloadAttempts = importFile.downloadAttempts + 1;
+
   if (!importFile.fileId) {
     // Generate a unique key for the file using the import ID, a UUID, and the file extension
     const uuid: string = uuidv4();
@@ -551,39 +559,35 @@ const logFileToBeCollected = async (
     await ImportFile.update(
       {
         fileId: fileRecord.id,
-        downloadAttempts: importFile.dataValues.downloadAttempts + 1,
+        downloadAttempts,
         status: IMPORT_STATUSES.COLLECTING,
       },
       {
         where: {
-          importId,
-          ftpFileInfo: {
-            [Op.contains]: {
-              path: availableFile.fileInfo.path,
-              name: availableFile.fileInfo.name,
-            },
-          },
+          id: importFile.id,
         },
         lock: true, // Lock the row for update to prevent race conditions
       },
     );
   } else {
+    // Step 2: Fetch the associated file record
+    const file = await File.findOne({
+      attributes: ['key'],
+      where: {
+        id: importFile.fileId,
+      },
+    });
+
     // Retrieve the key from the existing import file record
-    key = importFile.file.dataValues.key;
+    key = file ? file.key : null;
     await ImportFile.update(
       {
-        downloadAttempts: importFile.dataValues.downloadAttempts + 1,
+        downloadAttempts,
         status: IMPORT_STATUSES.COLLECTING,
       },
       {
         where: {
-          importId,
-          ftpFileInfo: {
-            [Op.contains]: {
-              path: availableFile.fileInfo.path,
-              name: availableFile.fileInfo.name,
-            },
-          },
+          id: importFile.id,
         },
         lock: true, // Lock the row for update to prevent race conditions
       },
@@ -593,7 +597,7 @@ const logFileToBeCollected = async (
   return {
     importFileId: importFile.id,
     key,
-    attempts: importFile.dataValues.downloadAttempts,
+    attempts: downloadAttempts,
   };
 };
 

diff --git a/src/lib/importSystem/tests/process.test.js b/src/lib/importSystem/tests/process.test.js
@@ -2,7 +2,6 @@
 import { DataTypes, Op } from 'sequelize';
 import { processRecords } from '../process';
 import XMLStream from '../../stream/xml';
-import db from '../../../models';
 import { modelForTable } from '../../modelUtils';
 
 // Mock the external modules
-Original file line number
+Diff line change
@@ Expand Up / @@ -12,3 +12,5 @@ hses.zip @@
     temp/
     tests/
     .tmp/
+    *.test.js
+    !*.sql