Merge pull request #1169 from alliance-genome/release/v0.21.0

Release/v0.21.0

.ebextensions/aws-autoscaling.config

@@ -1,6 +1,6 @@
 option_settings:
   aws:ec2:instances:
-    InstanceTypes: r5.large
+    InstanceTypes: m6i.2xlarge
   aws:autoscaling:asg:
     Availability Zones: Any
     MaxSize: 1

.github/workflows/github-actions-PR-validation.yml

@@ -75,10 +75,10 @@ jobs:
         run: |
           git fetch -q origin ${{ github.base_ref }} ${{ github.head_ref }}
           git diff --name-only origin/${{ github.base_ref }} origin/${{ github.head_ref }}
-      - name: Setup Node.js (v16)
+      - name: Setup Node.js (v18)
         uses: actions/setup-node@v3
         with:
-          node-version: 16
+          node-version: 18
       - name: Report Node version
         run: |
           node -v

Dockerfile

@@ -1,7 +1,7 @@
 ARG OVERWRITE_VERSION
 
 ### Stage 1: build UI
-FROM node:16 AS BUILD_UI_STAGE
+FROM node:18 AS BUILD_UI_STAGE
 
 WORKDIR /agr_curation
 COPY src/main/cliapp ./cliapp
@@ -48,4 +48,4 @@ ENV QUARKUS_HIBERNATE_SEARCH_ORM_ELASTICSEARCH_HOSTS opensearch:9200
 ENV QUARKUS_HIBERNATE_SEARCH_ORM_ELASTICSEARCH_PROTOCOL http
 
 # Start the application
-CMD ["java", "-Xmx15g", "-jar", "agr_curation_api-runner.jar"]
+CMD ["java", "-Xms30g", "-Xmx30g", "-jar", "agr_curation_api-runner.jar"]

README.md

@@ -28,6 +28,7 @@ These instructions will get you a copy of the project and the API up and running
    *  [Additional deployment steps](#additional-deployment-steps)
    *  [Release versioning](#release-versioning)
    *  [Release Creation](#release-creation)
+   *  [Database](#database)
 -  [Loading Data](#loading-data)
 -  [Submitting Data](#submitting-data)
 
@@ -443,27 +444,41 @@ after merging into the respective branch to trigger deployment and to ensure
 the new version of the application can function in a consistent state upon and after deployment.
 
 1. Compare the environment variables set in the Elastic Beanstalk environment between the environment you want to deploy to and from (e.g. compare curation-beta to curation-alpha for deployment to beta, or curation-production to curation-beta for deployment to production). This can be done through the [EB console](https://console.aws.amazon.com/elasticbeanstalk/home?region=us-east-1#/application/overview?applicationName=curation-app), or by using the `eb printenv` CLI. Scan for relevant change:
-   *  New variables should be added to the environment to be deployed to, **before** initiating the deployment
-   *  ENV-specific value changes should be ignored (for example, datasource host will be different for each)
-   *  Other variable value changes should be propagated as appropriate, **before** initiating the deployment
-   *  Removed variables should be cleaned up **after** successfull deployment
-2. Connect to the Environment's Elastic search domain by entering its domain endpoint in Cerebro, and delete all indexes.
-   The domain endpoint URL can be found through the [Amazon OpenSearch console](https://console.aws.amazon.com/esv3/home?region=us-east-1#opensearch/domains), the cerebro UI is available on the application server through HTTP at port 9000.
-3. When wanting to deploy a prerelease to the beta environment, reset the beta postgres DB and roll down the latest production DB backup
-   (see the [agr_db_backups README](https://github.com/alliance-genome/agr_db_backups#manual-invocation)).  
+    *  New variables should be added to the environment to be deployed to, **before** initiating the deployment
+    *  ENV-specific value changes should be ignored (for example, datasource host will be different for each)
+    *  Other variable value changes should be propagated as appropriate, **before** initiating the deployment
+    *  Removed variables should be cleaned up **after** successfull deployment
+2. Connect to the Environment's search domain and delete all indexes. A link to the Cerebro view into each environment's search indexes is available in the curation interface under `Other Links` > `Elastic Search UI` (VPN connection required).
+   Alternatively, you can reach this UI manually by browsing to the [AGR Cerebro interface](http://cerebro.alliancegenome.org:9000) and entering the environment's domain endpoint manually. The domain endpoint URL can be found through the [Amazon OpenSearch console](https://us-east-1.console.aws.amazon.com/aos/home?region=us-east-1#opensearch/domains).
+3. When wanting to deploy a prerelease to the beta environment, reset the beta postgres DB and roll down the latest production DB backup.  
    This must be done to catch any potentially problems that could be caused by new data available only on the production environment,
    before the code causing it would get deployed to the production environment.  
-   To ensure users or loads can not write to the database while it is being reloaded, stop the (beta) application before
-   initiating the DB roll-down and restart it once the DB roll-down completed.
-   ```bash
-   > eb ssh curation-beta -e 'ssh -i $AGR_SSH_PEM_FILE' #Connect to relevant application server
-   > cd /var/app/current
-   > sudo docker-compose stop  #Stop the application
-   #Trigger DB roll-down locally and wait for completion
-   > sudo docker-compose start #(Re)start the application
-   ```
+   The restore function automatically prevents users from writing to the database while it is being reloaded,
+   by temporarily making the target database read-only and restoring in a separated database before renaming.
+    1. For instructions on how to trigger the restore, see the [agr_db_backups README](https://github.com/alliance-genome/agr_db_backups#manual-invocation)
+    2. After the restore completed, restart the beta environment app-server to re-apply all flyway migrations
+       that were not yet present in the restored (production) database.
+       ```bash
+       > aws elasticbeanstalk restart-app-server --environment-name curation-beta
+       ```
+    3. Check the logs for errors after app-server restart, which could indicate a DB restore failure
+        and troubleshoot accordingly if necessary to fix any errors.
 4. Tag and create the release in git and gitHub, as described in the [Release creation](#release-creation) section.
-5. Check the logs for the environment that you're releasing too and ensure that all migrations complete successfully.
+5. Check the logs for the environment that you're releasing to and ensure the application started successfully
+   and that all flyway migrations completed successfully. If errors occured, troubleshoot and fix accordingly.
+    * If the application failed to start due to incompatible indexes (the indexes were not deleted before deployment),
+      delete the indexes and restart the app-server by running
+      ```bash
+      > aws elasticbeanstalk restart-app-server --environment-name curation-<beta|production>
+      ```
+    * If any of the migrations failed, try restarting the application once more,
+      or create a new release to fix the failing migration if the second restart did not resolve the issue:
+      1. Create a new `release/v*` branch from the beta or production branch as appropriate.
+          * Update to the next release candidate (`a` in `vx.y.z-rca`) for fixes to beta, using the beta branch as starting commit
+          * Update to the next patch release (`z` in `vx.y.z`) for fixes to production, using the production branch as starting commit
+      2. Fix the required migration files
+      3. Create a PR to merge the new `release/v*` branch back into beta/production
+      4. repeat the deployment process for this new release (candidate) from the [additional-deployment-steps](#additional-deployment-steps) step 5 (release creation).
 6. Reindex all data types by calling the `system/reindexeverything` endpoint with default arguments (found in the
    System Endpoints section in the swagger UI) and follow-up through the log server to check for progress and errors.
 7. Once reindexing completed, look at the dashboard page (where you deployed to)
@@ -525,6 +540,35 @@ Once published, github actions kicks in and the release will get deployed to the
 Completion of these deployments is reported in the #a-team-code slack channel. After receiving a successful deployment notification,
 continue the remaining steps described in the [additional deployment steps section](#additional-deployment-steps).
 
+### Database
+The Curation application connects to the postgres DB using a user called `curation_app`, which is member of a role called `curation_admins`.
+This role and user are used to ensure the database can be closed for all but admin users on initiating a DB restore,
+such that no accidential writes can happen to the postgres database during the restore process to ensure data integrity.
+When restoring to or creating a new postgres server, ensure the `curation_admins` role exists and has a member user called `curation_app`,
+ to ensure the database can be restored with the correct ownerships, and give the `curation_admins` role all permissions to
+ the curation database, the `public` schema, and all tables and sequences in it, and change the default privileges to allow
+ the curation_admins role all permissions on all newly created tables and sequences in the `public` schema.
+
+This can be achieved by connecting to the curation database using the admin (postgres) user (using `psql`)
+and executing the following queries:  
+```sql
+-- Create the role
+CREATE ROLE curation_admins;
+
+-- Create user (change the password)
+CREATE USER curation_app WITH PASSWORD '...';
+-- Grant role to user
+GRANT curation_admins TO curation_app;
+
+-- Grant required privileges to group (role)
+GRANT ALL ON DATABASE curation TO GROUP curation_admins;
+GRANT ALL ON SCHEMA public TO GROUP curation_admins;
+GRANT ALL ON ALL TABLES IN SCHEMA public TO GROUP curation_admins;
+GRANT ALL ON ALL SEQUENCES IN SCHEMA public TO GROUP curation_admins;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON TABLES TO GROUP curation_admins;
+ALTER DEFAULT PRIVILEGES IN SCHEMA public GRANT ALL ON SEQUENCES TO GROUP curation_admins;
+```
+
 ## Loading Data
 
 Assuming the API is running on localhost, check the Swagger interface for the correct endpoint to post data to. Here is an example of loading the Gene BGI file from the Alliance.

RELEASE-NOTES.md

@@ -2,6 +2,21 @@
 
 https://agr-jira.atlassian.net/wiki/spaces/ATEAM/overview
 
+## v0.21.0
+ * New features
+   * Duplicate Disease Annotation popup (SCRUM-2911)
+   * Updated display of ontology terms in Tree View (SCRUM-3021)
+   * Improved display of notes popups (SCRUM-3005)
+   * Switch to single consensus set of defining properties for disease annotations (SCRUM-3008)
+   * Loaded Vertebrate Trait (VT) Ontology (SCRUM-2453)
+   * Loaded Ascomycete Phenotype Ontology (APO) (SCRUM-2447)
+   * Loaded Measurement Method Ontology (MMO) (SCRUM-1206)
+   * Added Data Provider column to Alleles table (SCRUM-3006)
+   * Added Data Provider column to Genes and AGMs tables (SCRUM-3007)
+   * Prevented loading of duplicate notes (SCRUM-2828)
+ * Fixes and maintenance   
+   * Allele indexing optimizations (SCRUM-2982) 
+
 ## v0.20.1
  * Fixes
  	* Temporary endpoint to reset data provider on all disease annotations (SCRUM-3037)

docker/run_os_cluster

@@ -0,0 +1,23 @@
+docker rm -f os01 os02 os03 os04
+
+NODES=os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15,os16
+
+docker run -it --rm --net curation -p 9200:9200 -p 9300:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os00 --name os00 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9201:9200 -p 9301:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os01 --name os01 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9202:9200 -p 9302:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os02 --name os02 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9203:9200 -p 9303:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os03 --name os03 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9204:9200 -p 9304:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os04 --name os04 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9205:9200 -p 9305:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os05 --name os05 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9206:9200 -p 9306:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os06 --name os06 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9207:9200 -p 9307:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os07 --name os07 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9208:9200 -p 9308:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os08 --name os08 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9209:9200 -p 9309:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os09 --name os09 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9210:9200 -p 9310:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os10 --name os10 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9211:9200 -p 9311:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os11 --name os11 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9212:9200 -p 9312:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os12 --name os12 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9213:9200 -p 9313:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os13 --name os13 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9214:9200 -p 9314:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os14 --name os14 opensearchproject/opensearch:1.2.4
+docker run -it --rm --net curation -p 9215:9200 -p 9315:9300 -e plugins.security.disabled=true -e cluster.initial_master_nodes=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e discovery.seed_hosts=os00,os01,os02,os03,os04,os05,os06,os07,os08,os09,os10,os11,os12,os13,os14,os15 -e node.name=os15 --name os15 opensearchproject/opensearch:1.2.4
+
+#docker run -d --net curation -p 9200:9200 -p 9600:9600 -e "discovery.type=single-node" -e "DISABLE_SECURITY_PLUGIN=true" --name opensearch opensearchproject/opensearch:1.2.4
+#docker run --rm -p 8080:8080 --net curation -e QUARKUS_HIBERNATE_SEARCH_ORM_ELASTICSEARCH_HOSTS=os00:9200,os01:9200,os02:9200,os03:9200,os04:9200,os05:9200,os06:9200,os07:9200,os08:9200,os09:9200,os10:9200,os11:9200,os12:9200,os13:9200,os14:9200,os15:9200 -e OKTA_AUTHENTICATION=false -it 100225593120.dkr.ecr.us-east-1.amazonaws.com/agr_curation:v0.20.0-rc4-138-g134cba0d

pom.xml

@@ -358,6 +358,12 @@
 			<artifactId>reflections</artifactId>
 			<version>0.10.2</version>
 		</dependency>
+
+		<dependency>
+    		<groupId>org.jsoup</groupId>
+    		<artifactId>jsoup</artifactId>
+    		<version>1.13.1</version>
+		</dependency>
 	</dependencies>
 
 	<build>

src/main/cliapp/Makefile

@@ -6,6 +6,9 @@ all:
 run:
 	npm start
 
+runalpha:
+	export API_URL=https://alpha-curation.alliancegenome.org; make run; unset API_URL
+
 build:
 	npm run-script build
 

src/main/cliapp/package.json

@@ -9,6 +9,7 @@
     },
     "license": "MIT",
     "dependencies": {
+        "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
         "@fullcalendar/core": "^5.7.2",
         "@fullcalendar/daygrid": "^5.7.2",
         "@fullcalendar/interaction": "^5.7.2",
@@ -61,6 +62,7 @@
         "not op_mini all"
     ],
     "devDependencies": {
+        "@babel/plugin-proposal-private-property-in-object": "^7.21.11",
         "msw": "^0.48.1"
     }
 }

src/main/cliapp/src/App.scss

@@ -4,4 +4,4 @@
 
 .p-datatable-scrollable .p-datatable-thead{
 	z-index : 2
-}
+}

src/main/cliapp/src/assets/layout/sass/_content.scss

@@ -10,3 +10,19 @@
 .layout-main {
     flex: 1 1 auto;
 }
+
+.wrap-word {
+    overflow-wrap : break-word;
+}
+
+.max-w-25rem {
+	max-width : 25rem;
+}
+
+.max-w-35rem {
+	max-width : 35rem;
+}
+
+.max-w-40rem {
+	max-width : 40rem;
+}

src/main/cliapp/src/components/DuplicationAction.js

@@ -0,0 +1,4 @@
+import { Button } from "primereact/button";
+export const DuplicationAction = ({ props, handleDuplication, disabled }) => {
+  return <Button icon="pi pi-copy" disabled={disabled} className="p-button-text" onClick={() => handleDuplication(props)}/> ;
+}

src/main/cliapp/src/components/Filters/FilterComponentMultiSelect.js

@@ -23,6 +23,7 @@ export function FilterComponentMultiSelect({ isEnabled, filterConfig, currentFil
 								 });
 						 }
 				 };
+				 tmp.sort((a, b) => a.optionLabel > b.optionLabel ? 1 : -1);
 				 setSelectableOptions(tmp);
 				 if(currentFilters && currentFilters[fieldSet.filterName]) {
 						 let newSelectedOptions = [];