Replace some CLI flags with args, fix checks not being added to assem…

…bly (#471) * bug: filter conflicting keys when adding jb config * Make input files args not flags in CLI * Make "not-editable" the default when adding fasta * Remove external dependency from test * bug: keep file from being unzipped multiple times * Add default checks to assembly when it is created * Fix dev env ontology location * Remove unused import * Set assembly status to 0 so feature checks work * Fix CLI tests
GMOD · Oct 22, 2024 · 0fc65ed · 0fc65ed
1 parent 7cc023b
commit 0fc65ed
Show file tree

Hide file tree

Showing 32 changed files with 327 additions and 325 deletions.
diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml
@@ -40,6 +40,7 @@ jobs:
         working-directory: packages/jbrowse-plugin-apollo
       - name: Start apollo
         run: |
+          yarn --cwd packages/apollo-collaboration-server/test/data/ serve --listen 3131 &
           yarn --cwd packages/apollo-shared start &
           ALLOW_ROOT_USER=true ROOT_USER_NAME=admin ROOT_USER_PASSWORD=pass yarn --cwd packages/apollo-collaboration-server start &
       - name: Run CLI tests

diff --git a/.github/workflows/reload_demo_data.yml b/.github/workflows/reload_demo_data.yml
@@ -89,23 +89,21 @@ jobs:
             /data/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.genomic.fa.gz \
             --fai /data/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.genomic.fa.gz.fai \
             --gzi /data/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.genomic.fa.gz.gzi \
-            --assembly 'Schistosoma haematobium' \
-            --not-editable
+            --assembly 'Schistosoma haematobium'
 
           apollo assembly \
             add-from-fasta \
             /data/schistosoma_mansoni.PRJEA36577.WBPS19.genomic.fa.gz \
             --fai /data/schistosoma_mansoni.PRJEA36577.WBPS19.genomic.fa.gz.fai \
             --gzi /data/schistosoma_mansoni.PRJEA36577.WBPS19.genomic.fa.gz.gzi \
-            --assembly 'Schistosoma mansoni' \
-            --not-editable
+            --assembly 'Schistosoma mansoni'
 
           apollo feature \
             import \
-            --input-file /data/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.annotations.genes.gff3 \
+            /data/schistosoma_haematobium.TD2_PRJEB44434.WBPS19.annotations.genes.gff3 \
             --assembly 'Schistosoma haematobium'
 
           apollo feature \
             import \
-            --input-file /data/schistosoma_mansoni.PRJEA36577.WBPS19.annotations.genes.gff3 \
+            /data/schistosoma_mansoni.PRJEA36577.WBPS19.annotations.genes.gff3 \
             --assembly 'Schistosoma mansoni'
diff --git a/packages/apollo-cli/README.md b/packages/apollo-cli/README.md
@@ -47,18 +47,18 @@ USAGE
 - [`apollo feature edit-type`](#apollo-feature-edit-type)
 - [`apollo feature get`](#apollo-feature-get)
 - [`apollo feature get-id`](#apollo-feature-get-id)
-- [`apollo feature import`](#apollo-feature-import)
+- [`apollo feature import INPUT-FILE`](#apollo-feature-import-input-file)
 - [`apollo feature search`](#apollo-feature-search)
 - [`apollo file delete`](#apollo-file-delete)
 - [`apollo file download`](#apollo-file-download)
 - [`apollo file get`](#apollo-file-get)
-- [`apollo file upload`](#apollo-file-upload)
+- [`apollo file upload INPUT-FILE`](#apollo-file-upload-input-file)
 - [`apollo help [COMMANDS]`](#apollo-help-commands)
 - [`apollo jbrowse get-config`](#apollo-jbrowse-get-config)
 - [`apollo jbrowse set-config INPUTFILE`](#apollo-jbrowse-set-config-inputfile)
 - [`apollo login`](#apollo-login)
 - [`apollo logout`](#apollo-logout)
-- [`apollo refseq add-alias`](#apollo-refseq-add-alias)
+- [`apollo refseq add-alias INPUT-FILE`](#apollo-refseq-add-alias-input-file)
 - [`apollo refseq get`](#apollo-refseq-get)
 - [`apollo status`](#apollo-status)
 - [`apollo user get`](#apollo-user-get)
@@ -69,21 +69,20 @@ Add a new assembly from fasta input
 
 ```
 USAGE
-  $ apollo assembly add-from-fasta INPUT [--profile <value>] [--config-file <value>] [-a <value>] [-x <value>] [-f] [-n]
-    [--fai <value>] [--gzi <value>] [-z | -d]
+  $ apollo assembly add-from-fasta INPUT [--profile <value>] [--config-file <value>] [-a <value>] [-f] [-e] [--fai <value>]
+    [--gzi <value>] [-z | -d]
 
 ARGUMENTS
-  INPUT  Input fasta file, local or remote, or id of a previously uploaded file
+  INPUT  Input fasta file, local or remote, or id of a previously uploaded file. For local or remote files, it is
+         assumed the file is bgzip'd with `bgzip` and indexed with `samtools faidx`. The indexes are assumed to be at
+         <my.fasta.gz>.fai and <my.fasta.gz>.gzi unless the options --fai and --gzi are provided.
 
 FLAGS
   -a, --assembly=<value>     Name for this assembly. Use the file name if omitted
   -d, --decompressed         For local file input: Override autodetection and instruct that input is decompressed
+  -e, --editable             Instead of using indexed fasta lookup, the sequence is loaded into the Apollo database and
+                             is editable. Use with caution, as editing the sequence often has unintended side effects.
   -f, --force                Delete existing assembly, if it exists
-  -n, --not-editable         The fasta sequence is not editable. Apollo will not load it into the database and instead
-                             use the provided indexes to query it. This option assumes the fasta file is bgzip'd with
-                             `bgzip` and indexed with `samtools faidx`. Indexes should be named <my.fasta.gz>.gzi and
-                             <my.fasta.gz>.fai unless options --fai and --gzi are set
-  -x, --index=<value>        URL of the index. Required if input is an external source
   -z, --gzip                 For local file input: Override autodetection and instruct that input is gzip compressed
       --config-file=<value>  Use this config file (mostly for testing)
       --fai=<value>          Fasta index of the (not-editable) fasta file
@@ -137,7 +136,7 @@ DESCRIPTION
 EXAMPLES
   Import sequences and features:
 
-    $ apollo assembly add-from-gff -i genome.gff -a myAssembly
+    $ apollo assembly add-from-gff genome.gff -a myAssembly
 
   Import sequences only:
 
@@ -674,18 +673,20 @@ EXAMPLES
 _See code:
 [src/commands/feature/get-id.ts](https://github.com/GMOD/Apollo3/blob/v0.1.20/packages/apollo-cli/src/commands/feature/get-id.ts)_
 
-## `apollo feature import`
+## `apollo feature import INPUT-FILE`
 
 Import features from local gff file
 
 ```
 USAGE
-  $ apollo feature import -i <value> -a <value> [--profile <value>] [--config-file <value>] [-d]
+  $ apollo feature import INPUT-FILE -a <value> [--profile <value>] [--config-file <value>] [-d]
+
+ARGUMENTS
+  INPUT-FILE  Input gff file
 
 FLAGS
   -a, --assembly=<value>     (required) Import into this assembly name or assembly ID
   -d, --delete-existing      Delete existing features before importing
-  -i, --input-file=<value>   (required) Input gff file
       --config-file=<value>  Use this config file (mostly for testing)
       --profile=<value>      Use credentials from this profile
 
@@ -697,7 +698,7 @@ DESCRIPTION
 EXAMPLES
   Delete features in myAssembly and then import features.gff3:
 
-    $ apollo feature import -d -i features.gff3 -a myAssembly
+    $ apollo feature import features.gff3 -d -a myAssembly
 ```
 
 _See code:
@@ -836,18 +837,20 @@ EXAMPLES
 _See code:
 [src/commands/file/get.ts](https://github.com/GMOD/Apollo3/blob/v0.1.20/packages/apollo-cli/src/commands/file/get.ts)_
 
-## `apollo file upload`
+## `apollo file upload INPUT-FILE`
 
 Upload a local file to the Apollo server
 
 ```
 USAGE
-  $ apollo file upload -i <value> [--profile <value>] [--config-file <value>] [-t
+  $ apollo file upload INPUT-FILE [--profile <value>] [--config-file <value>] [-t
     text/x-fasta|text/x-gff3|application/x-bgzip-fasta|text/x-fai|application/x-gzi] [-z | -d]
 
+ARGUMENTS
+  INPUT-FILE  Local file to upload
+
 FLAGS
   -d, --decompressed         Override autodetection and instruct that input is decompressed
-  -i, --input-file=<value>   (required) Local file to upload
   -t, --type=<option>        Set file type or autodetected it if not set.
                              NB: There is no check for whether the file complies to this type
                              <options: text/x-fasta|text/x-gff3|application/x-bgzip-fasta|text/x-fai|application/x-gzi>
@@ -865,7 +868,7 @@ DESCRIPTION
 EXAMPLES
   Upload local file, type auto-detected:
 
-    $ apollo file upload -i genome.fa > file.json
+    $ apollo file upload genome.fa > file.json
 ```
 
 _See code:
@@ -1016,17 +1019,19 @@ EXAMPLES
 _See code:
 [src/commands/logout.ts](https://github.com/GMOD/Apollo3/blob/v0.1.20/packages/apollo-cli/src/commands/logout.ts)_
 
-## `apollo refseq add-alias`
+## `apollo refseq add-alias INPUT-FILE`
 
 Add reference name aliases from a file
 
 ```
 USAGE
-  $ apollo refseq add-alias -i <value> -a <value> [--profile <value>] [--config-file <value>]
+  $ apollo refseq add-alias INPUT-FILE -a <value> [--profile <value>] [--config-file <value>]
+
+ARGUMENTS
+  INPUT-FILE  Input refname alias file
 
 FLAGS
   -a, --assembly=<value>     (required) Name for this assembly.
-  -i, --input-file=<value>   (required) Input refname alias file
       --config-file=<value>  Use this config file (mostly for testing)
       --profile=<value>      Use credentials from this profile
 
@@ -1039,7 +1044,7 @@ DESCRIPTION
 EXAMPLES
   Add reference name aliases:
 
-    $ apollo refseq add-alias -i alias.txt -a myAssembly
+    $ apollo refseq add-alias alias.txt -a myAssembly
 ```
 
 _See code:

diff --git a/packages/apollo-cli/src/commands/assembly/add-from-fasta.ts b/packages/apollo-cli/src/commands/assembly/add-from-fasta.ts
@@ -7,11 +7,15 @@ import { ObjectId } from 'bson'
 import { FileCommand } from '../../fileCommand.js'
 import { queryApollo, submitAssembly } from '../../utils.js'
 import { Response } from 'undici'
+import {
+  SerializedAddAssemblyFromExternalChange,
+  SerializedAddAssemblyFromFileChange,
+} from '@apollo-annotation/shared'
 
 export default class AddFasta extends FileCommand {
   static summary = 'Add a new assembly from fasta input'
   static description = `Add new assembly. The input fasta may be:
-    * A local file 
+    * A local file
     * An external fasta file
     * The id of a file previously uploaded to Apollo`
 
@@ -30,7 +34,11 @@ export default class AddFasta extends FileCommand {
   static args = {
     input: Args.string({
       description:
-        'Input fasta file, local or remote, or id of a previously uploaded file',
+        "Input fasta file, local or remote, or id of a previously uploaded \
+file. For local or remote files, it is assumed the file is bgzip'd with \
+`bgzip` and indexed with `samtools faidx`. The indexes are assumed to be at \
+<my.fasta.gz>.fai and <my.fasta.gz>.gzi unless the options --fai and --gzi are \
+provided.",
       required: true,
     }),
   }
@@ -40,21 +48,16 @@ export default class AddFasta extends FileCommand {
       char: 'a',
       description: 'Name for this assembly. Use the file name if omitted',
     }),
-    index: Flags.string({
-      char: 'x',
-      description: 'URL of the index. Required if input is an external source',
-    }),
     force: Flags.boolean({
       char: 'f',
       description: 'Delete existing assembly, if it exists',
     }),
-    'not-editable': Flags.boolean({
-      char: 'n',
+    editable: Flags.boolean({
+      char: 'e',
       description:
-        "The fasta sequence is not editable. Apollo will not load it into \
-the database and instead use the provided indexes to query it. This option assumes \
-the fasta file is bgzip'd with `bgzip` and indexed with `samtools faidx`. \
-Indexes should be named <my.fasta.gz>.gzi and <my.fasta.gz>.fai unless options --fai and --gzi are set",
+        'Instead of using indexed fasta lookup, the sequence is loaded into \
+the Apollo database and is editable. Use with caution, as editing the sequence \
+often has unintended side effects.',
     }),
     fai: Flags.string({
       description: 'Fasta index of the (not-editable) fasta file',
@@ -77,8 +80,7 @@ Indexes should be named <my.fasta.gz>.gzi and <my.fasta.gz>.fai unless options -
   }
 
   public async run(): Promise<void> {
-    const { args } = await this.parse(AddFasta)
-    const { flags } = await this.parse(AddFasta)
+    const { args, flags } = await this.parse(AddFasta)
 
     const access = await this.getAccess()
 
@@ -91,22 +93,46 @@ Indexes should be named <my.fasta.gz>.gzi and <my.fasta.gz>.fai unless options -
     )
     const isExternal = isValidHttpUrl(args.input)
 
-    let body
+    let body:
+      | SerializedAddAssemblyFromFileChange
+      | SerializedAddAssemblyFromExternalChange
     if (isExternal) {
-      if (flags.index === undefined) {
-        this.error(
-          'Please provide the URL to the index of the external fasta file',
-        )
-      }
+      const fai = flags.fai ?? `${args.input}.fai`
+      const gzi = flags.gzi ?? `${args.input}.gzi`
       body = {
         assemblyName,
         typeName: 'AddAssemblyFromExternalChange',
-        externalLocation: {
-          fa: args.input,
-          fai: flags.index,
-        },
+        externalLocation: { fa: args.input, fai, gzi },
+        assembly: new ObjectId().toHexString(),
+      }
+    } else if (flags.editable) {
+      if (!isExternal && !fs.existsSync(args.input) && !fastaIsFileId) {
+        this.error(`Input "${args.input}" is not valid`)
+      }
+      let isGzip = args.input.endsWith('.gz')
+      if (flags.gzip) {
+        isGzip = true
       }
-    } else if (flags['not-editable']) {
+      if (flags.decompressed) {
+        isGzip = false
+      }
+
+      const fileId = fastaIsFileId
+        ? args.input
+        : await this.uploadFile(
+            access.address,
+            access.accessToken,
+            args.input,
+            'text/x-fasta',
+            isGzip,
+          )
+      body = {
+        assemblyName,
+        fileIds: { fa: fileId },
+        typeName: 'AddAssemblyFromFileChange',
+        assembly: new ObjectId().toHexString(),
+      }
+    } else {
       const gzi = flags.gzi ?? `${args.input}.gzi`
       const fai = flags.fai ?? `${args.input}.fai`
 
@@ -165,41 +191,10 @@ Indexes should be named <my.fasta.gz>.gzi and <my.fasta.gz>.fai unless options -
       body = {
         assemblyName,
         typeName: 'AddAssemblyFromFileChange',
-        fileIds: {
-          fa: faId,
-          fai: faiId,
-          gzi: gziId,
-        },
-      }
-    } else {
-      if (!isExternal && !fs.existsSync(args.input) && !fastaIsFileId) {
-        this.error(`Input "${args.input}" is not valid`)
-      }
-      let isGzip = args.input.endsWith('.gz')
-      if (flags.gzip) {
-        isGzip = true
-      }
-      if (flags.decompressed) {
-        isGzip = false
-      }
-
-      const fileId = fastaIsFileId
-        ? args.input
-        : await this.uploadFile(
-            access.address,
-            access.accessToken,
-            args.input,
-            'text/x-fasta',
-            isGzip,
-          )
-      body = {
-        assemblyName,
-        fileIds: { fa: fileId },
-        typeName: 'AddAssemblyFromFileChange',
+        fileIds: { fa: faId, fai: faiId, gzi: gziId },
         assembly: new ObjectId().toHexString(),
       }
     }
-
     const rec = await submitAssembly(
       access.address,
       access.accessToken,