Pull grant and grantee data from HSES on a schedule

.circleci/config.yml

@@ -70,6 +70,15 @@ commands:
         description: "Name of CircleCI project environment variable that
           holds the New Relic License key, a required application variable"
         type: env_var_name
+      hses_data_file_url:
+        description: "Url to download HSES grants and grantee data from"
+        type: env_var_name
+      hses_data_username:
+        description: "Username used to access the HSES grants and grantee data"
+        type: env_var_name
+      hses_data_password:
+        description: "Password used to access the HSES grants and grantee data"
+        type: env_var_name
     steps:
       - run:
           name: Login with service account
@@ -86,7 +95,10 @@ commands:
               --var AUTH_CLIENT_ID=${<< parameters.auth_client_id >>} \
               --var AUTH_CLIENT_SECRET=${<< parameters.auth_client_secret >>} \
               --var NEW_RELIC_LICENSE_KEY=${<< parameters.new_relic_license >>} \
-              --var SESSION_SECRET=${<< parameters.session_secret >>}
+              --var SESSION_SECRET=${<< parameters.session_secret >>} \
+              --var HSES_DATA_FILE_URL=${<< parameters.hses_data_file_url >>} \
+              --var HSES_DATA_USERNAME=${<< parameters.hses_data_username >>} \
+              --var HSES_DATA_PASSWORD=${<< parameters.hses_data_password >>}
 parameters:
   cg_org:
     description: "Cloud Foundry cloud.gov organization name"
@@ -121,7 +133,7 @@ parameters:
     default: "main"
     type: string
   sandbox_git_branch:  # change to feature branch to test deployment
-    default: "js-fix-user-autoinc"
+    default: "kw-pull-hs-data"
     type: string
 jobs:
   build_and_lint:
@@ -168,6 +180,9 @@ jobs:
       - run:
           name: Run migrations ci
           command: yarn db:migrate:ci
+      - run:
+          name: Run seeders
+          command: yarn db:seed:ci
       - run:
           name: Test backend
           command: yarn test:ci
@@ -302,6 +317,9 @@ jobs:
                 deploy_config_file: deployment_config/sandbox_vars.yml
                 new_relic_license: NEW_RELIC_LICENSE_KEY
                 session_secret: SANDBOX_SESSION_SECRET
+                hses_data_file_url: HSES_DATA_FILE_URL
+                hses_data_username: HSES_DATA_USERNAME
+                hses_data_password: HSES_DATA_PASSWORD
             - run:
                 name: Return database to neutral, then migrate and seed
                 command: |
@@ -323,6 +341,9 @@ jobs:
                 deploy_config_file: deployment_config/dev_vars.yml
                 new_relic_license: NEW_RELIC_LICENSE_KEY
                 session_secret: DEV_SESSION_SECRET
+                hses_data_file_url: HSES_DATA_FILE_URL
+                hses_data_username: HSES_DATA_USERNAME
+                hses_data_password: HSES_DATA_PASSWORD
             - run:
                 name: Undo database seeding, then migrate and seed
                 command: |
@@ -343,6 +364,9 @@ jobs:
                 deploy_config_file: deployment_config/staging_vars.yml
                 new_relic_license: NEW_RELIC_LICENSE_KEY
                 session_secret: STAGING_SESSION_SECRET
+                hses_data_file_url: HSES_DATA_FILE_URL
+                hses_data_username: HSES_DATA_USERNAME
+                hses_data_password: HSES_DATA_PASSWORD
             - run:
                 name: Run database migrations
                 command: |
@@ -363,6 +387,9 @@ jobs:
                 deploy_config_file: deployment_config/prod_vars.yml
                 new_relic_license: PROD_NEW_RELIC_LICENSE_KEY
                 session_secret: PROD_SESSION_SECRET
+                hses_data_file_url: HSES_DATA_FILE_URL
+                hses_data_username: HSES_DATA_USERNAME
+                hses_data_password: HSES_DATA_PASSWORD
             - run:
                 name: Run database migrations
                 command: |

.env.example

@@ -10,7 +10,9 @@ POSTGRES_USERNAME=postgres
 POSTGRES_PASSWORD=something_secret
 POSTGRES_DB=ttasmarthub
 POSTGRES_HOST=localhost
+# Add Oauth client id for local development from "Development Credentials" document here.
 AUTH_CLIENT_ID=clientId
+# Add Oauth client secret for local development from "Development Credentials" document here.
 AUTH_CLIENT_SECRET=clientSecret
 SESSION_SECRET=secret
 TTA_SMART_HUB_URI=http://localhost:3000
@@ -19,11 +21,14 @@ AUTH_BASE=https://uat.hsesinfo.org
 REDIRECT_URI_HOST=http://localhost:8080
 # CURRENT_USER_ID controls the logged in user when BYPASS_AUTH is set to true.
 # This only works in non-production environments
-CURRENT_USER_ID=1;
+CURRENT_USER_ID=1
 # NEW_RELIC_LICENSE_KEY can be omitted in local development
 NEW_RELIC_LICENSE_KEY=secret_key
 # Set to false to require user to go through auth flow, never true in production envs
 BYPASS_AUTH=true
+HSES_DATA_FILE_URL=url
+HSES_DATA_USERNAME=username
+HSES_DATA_PASSWORD=password
 # In production, Sequelize instance is created with a postgres URI.
 # This URI is automatically dropped into the cloud.gov environment as the env variable DATABASE_URL
 DATABASE_URL=secret

README.md

@@ -9,17 +9,27 @@ For the latest on our product mission, goals, initiatives, and KPIs, see the [Pr
 ## Getting Started
 
 ### Set up
-
-Make sure Docker is installed. To check run `docker ps`
-
-Run `yarn docker:deps`. This builds the frontend and backend docker containers and install dependencies. You only need to run this step the first time you fire up the app and when dependencies are added/updated/removed. Running `yarn docker:start` starts the backend and frontend, browse to `http://localhost:3000` to hit the frontend and `http://localhost:3000/api` to hit the backend. Copying `.env.example` to `.env`, substituting in your user id and group id will cause any files created in docker containers to be owned by your user on your host.
-
-You can also run build commands directly on your host (without docker). Make sure you install dependencies when changing execution method. You could see some odd errors if you install dependencies for docker and then run yarn commands directly on the host, especially if you are developing on windows. If you want to use the host yarn commands be sure to run `yarn deps:local` before any other yarn commands. Likewise if you want to use docker make sure you run `yarn docker:deps`.
+#### Docker
+
+1. Make sure Docker is installed. To check run `docker ps`.
+2. Make sure you have Node 12.20.0 installed.
+3. Run `yarn docker:deps`. This builds the frontend and backend docker containers and install dependencies. You only need to run this step the first time you fire up the app and when dependencies are added/updated/removed. 
+4. Copy `.env.example` to `.env`.  
+6. Change the `AUTH_CLIENT_ID` and `AUTH_CLIENT_SECRET` variables to to values found in the "Values for local development" section of the "Development Credentials" document. If you don't have access to this document, please ask in the hs-vendors-ohs-tta channel of the gsa-tts slack channel.
+7. Optionally, set `CURRENT_USER` to your current user's uid:gid. This will cause files created by docker compose to be owned by your user instead of root.
+8. Run `yarn docker:db:migrate` to run DB migrations
+9. Run `yarn docker:db:seed` to seed the database with test data.
+10. Run `yarn docker:start` to start the application. The frontend will be available on `localhost:3000` and the backend will run on `localhost:8080`.
+11. Run `yarn docker:stop` to stop the servers and remove the docker containers.
 
 The frontend [proxies requests](https://create-react-app.dev/docs/proxying-api-requests-in-development/) to paths it doesn't recognize to the backend.
 
 Api documentation uses [Redoc](https://github.com/Redocly/redoc) to serve documentation files. These files can be found in the `docs/openapi` folder. Api documentation should be split into separate files when appropriate to prevent huge hard to grasp yaml files.
 
+#### Local build
+
+You can also run build commands directly on your host (without docker). Make sure you install dependencies when changing execution method. You could see some odd errors if you install dependencies for docker and then run yarn commands directly on the host, especially if you are developing on windows. If you want to use the host yarn commands be sure to run `yarn deps:local` before any other yarn commands. Likewise if you want to use docker make sure you run `yarn docker:deps`.
+
 ### Running Tests
 
 Run `yarn docker:deps` to install dependencies. Run `yarn docker:db:migrate` and `yarn docker:test` to run all tests for the frontend and backend.
@@ -215,6 +225,18 @@ Our project includes four deployed Postgres databases, one to interact with each
 
 You can run psql commands directly against a deployed database by following these directions.
 
+1. Install Cloud Foundry CLI tool
+
+    - On MacOS: `brew install cloudfoundry/tap/cf-cli`
+    - On other platforms: [Download and install cf][cf-install]
+
+1. Login to cloud.gov account
+
+    ```bash
+    cf login -a api.fr.cloud.gov --sso
+    # follow temporary authorization code prompts
+    ```
+
 1. Install the cloud foundry plugin [cf-service-connect][cf-service-connect]
 
 	```bash
@@ -247,6 +269,7 @@ You can run psql commands directly against a deployed database by following thes
 [circleci-envvar]: https://app.circleci.com/settings/project/github/adhocteam/Head-Start-TTADP/environment-variables?return-to=https%3A%2F%2Fcircleci.com%2Fdashboard
 [cloudgov]: https://dashboard.fr.cloud.gov/home
 [cloudgov-deployer]: https://cloud.gov/docs/services/cloud-gov-service-account/
+[cf-install]: https://docs.cloudfoundry.org/cf-cli/install-go-cli.html
 [cf-service-connect]: https://github.com/cloud-gov/cf-service-connect
 [hhs-main]: https://github.com/HHS/Head-Start-TTADP/tree/main
 [hhs-prod]: https://github.com/HHS/Head-Start-TTADP/tree/production

manifest.yml

@@ -16,6 +16,9 @@ applications:
       REDIRECT_URI_HOST: ((REDIRECT_URI_HOST))
       SESSION_SECRET: ((SESSION_SECRET))
       TTA_SMART_HUB_URI: ((TTA_SMART_HUB_URI))
+      HSES_DATA_FILE_URL: ((HSES_DATA_FILE_URL))
+      HSES_DATA_USERNAME: ((HSES_DATA_USERNAME))
+      HSES_DATA_PASSWORD: ((HSES_DATA_PASSWORD))
     services:
       - ((rds_instance))
       - ((s3_doc_upload_bucket))

package.json

@@ -132,9 +132,11 @@
   "dependencies": {
     "@babel/runtime": "^7.12.1",
     "@cucumber/cucumber": "^7.0.0-rc.0",
+    "adm-zip": "^0.5.1",
     "axios": "^0.21.1",
     "chromedriver": "^87.0.0",
     "client-oauth2": "^4.3.3",
+    "cron": "^1.8.2",
     "csv-parse": "^4.14.1",
     "cucumber-html-reporter": "^5.2.0",
     "dotenv": "^8.2.0",
@@ -146,6 +148,7 @@
     "http-codes": "^1.0.0",
     "lodash": "^4.17.20",
     "memorystore": "^1.6.2",
+    "mz": "^2.7.0",
     "newrelic": "^7.0.1",
     "pg": "^8.3.3",
     "puppeteer": "^5.3.1",
@@ -154,6 +157,7 @@
     "sequelize-cli": "^6.2.0",
     "url-join": "^4.0.1",
     "winston": "^3.3.3",
+    "xml2json": "^0.12.0",
     "yargs": "^16.1.1"
   }
 }

src/app.js

@@ -7,7 +7,9 @@ import memorystore from 'memorystore';
 import path from 'path';
 import join from 'url-join';
 import { INTERNAL_SERVER_ERROR } from 'http-codes';
+import { CronJob } from 'cron';
 import { hsesAuth } from './middleware/authMiddleware';
+import updateGrantsGrantees from './lib/updateGrantsGrantees';
 
 import findOrCreateUser from './services/accessValidation';
 
@@ -77,4 +79,23 @@ if (process.env.NODE_ENV === 'production') {
   });
 }
 
+// Set timing parameters.
+// Run at midnight
+const schedule = '0 0 * * *';
+const timezone = 'America/New_York';
+
+const runJob = () => {
+  try {
+    updateGrantsGrantees();
+  } catch (error) {
+    logger.error(`Error processing HSES file: ${error}`);
+  }
+};
+
+// Run only on one instance
+if (process.env.CF_INSTANCE_INDEX === '0') {
+  const job = new CronJob(schedule, () => runJob(), null, true, timezone);
+  job.start();
+}
+
 module.exports = app;

src/lib/updateGrantsGrantees.js

@@ -0,0 +1,124 @@
+import AdmZip from 'adm-zip';
+import { toJson } from 'xml2json';
+import {} from 'dotenv/config';
+import axios from 'axios';
+import {
+  Grantee, Grant,
+} from '../models';
+import logger from '../logger';
+
+const fs = require('mz/fs');
+/**
+ * Reads HSES data files that were previously extracted to the "temp" directory.
+ * The files received from HSES are:
+ *
+ * agency.xml - grantee and grantee that are delegates
+ * grant_agency.xml - junction between grants and agencies
+ * grant_award.xml - grants
+ * grant_award_replacement.xml
+ * grant_program.xml
+ *
+ * The grantee data is them filtered to exclude delegates
+ *
+ */
+export async function processFiles() {
+  let grantGrantees;
+  let grants;
+  const granteesForDb = [];
+  const grantsForDb = [];
+
+  try {
+    const grantAgencyData = await fs.readFile('./temp/grant_agency.xml');
+    const json = toJson(grantAgencyData);
+    const grantAgency = JSON.parse(json);
+    // we are only interested in non-delegates
+    grantGrantees = grantAgency.grant_agencies.grant_agency.filter(
+      (g) => g.grant_agency_number === '0',
+    );
+
+    // process grantees aka agencies that are non-delegates
+    const agencyData = await fs.readFile('./temp/agency.xml');
+    const agency = JSON.parse(toJson(agencyData));
+
+    // filter out delegates by matching to the non-delegates
+    // eslint-disable-next-line max-len
+    const granteesNonDelegates = agency.agencies.agency.filter((a) => grantGrantees.some((gg) => gg.agency_id === a.agency_id));
+
+    const hubGranteeIds = await Grantee.findAll({ attributes: ['id'] }).map((hgi) => hgi.id);
+
+    // process grants
+    const grantData = await fs.readFile('./temp/grant_award.xml');
+    const grant = JSON.parse(toJson(grantData));
+
+    // Check if the grantee id already exists in the smarthub db OR if it belongs to
+    // at least one active grant. grant_award data structure includes agency_id
+    // eslint-disable-next-line max-len
+    const grantees = granteesNonDelegates.filter((gnd) => hubGranteeIds.some((id) => id.toString() === gnd.agency_id)
+    || grant.grant_awards.grant_award.some((ga) => ga.agency_id === gnd.agency_id && ga.grant_status === 'Active'));
+
+    grantees.forEach((g) => granteesForDb.push({
+      id: parseInt(g.agency_id, 10),
+      name: g.agency_name,
+    }));
+
+    await Grantee.bulkCreate(granteesForDb,
+      {
+        updateOnDuplicate: ['name', 'updatedAt'],
+      });
+
+    const hubGrantIds = await Grant.findAll({ attributes: ['id'] }).map((hgi) => hgi.id);
+
+    grants = grant.grant_awards.grant_award.filter((ga) => hubGrantIds.some((id) => id.toString() === ga.grant_award_id) || ga.grant_status === 'Active');
+
+    grants.forEach((g) => grantsForDb.push({
+      id: parseInt(g.grant_award_id, 10),
+      number: g.grant_number,
+      regionId: parseInt(g.region_id, 10),
+      granteeId: parseInt(g.agency_id, 10),
+      status: g.grant_status,
+      startDate: g.grant_start_date,
+      endDate: g.grant_end_date,
+    }));
+
+    await Grant.bulkCreate(grantsForDb,
+      {
+        updateOnDuplicate: ['number', 'regionId', 'granteeId', 'status', 'startDate', 'endDate', 'updatedAt'],
+      });
+  } catch (error) {
+    logger.error(`Error reading or updating database on HSES data import: ${error.message}`);
+    throw error;
+  }
+}
+
+// reading archives
+const zip = new AdmZip('./hses.zip');
+
+/**
+ * Downloads the HSES grantee/grant zip, extracts to the "temp" directory
+ * and calls processFiles to parse xml data and populate the Smart Hub db
+ *
+ * Note - file download needs to happen in deployed environments
+ */
+export default async function updateGrantsGrantees() {
+  try {
+    if (process.env.NODE_ENV === 'production') {
+      const response = await axios(process.env.HSES_DATA_FILE_URL, {
+        method: 'get',
+        url: process.env.HSES_DATA_FILE_URL,
+        responseType: 'stream',
+        auth: {
+          username: process.env.HSES_DATA_USERNAME,
+          password: process.env.HSES_DATA_PASSWORD,
+        },
+      });
+
+      await response.data.pipe(fs.createWriteStream('hses.zip'));
+    }
+    // extract to target path. Pass true to overwrite
+    zip.extractAllTo('./temp', true);
+
+    await processFiles();
+  } catch (error) {
+    logger.error(error);
+  }
+}

src/migrations/20201029214432-add-title-to-user.js

@@ -23,6 +23,7 @@ module.exports = {
         'homeRegionId',
         {
           type: Sequelize.INTEGER,
+          allowNull: true,
           references: {
             model: 'Regions',
             key: 'id',