Add a general Repository abstraction for remote and local repos

- Move repository.py -> repositories/local.py (LocalRepository)
- Move remote.py -> repositories/remote.py
- Add repository.py with Repository abstraction that proxies access
  to the above two classes
- Have RPCErrors act like normal Errors so that general code doesn't
  have to know its particulars
- Likewise, handle servers too old for nonce support internally to
  RemoteRepository
- Have the Repository object say whether its backend is remote or not
  rather than hardcoding knowledge from class names

docs/internals/repositories.rst

@@ -0,0 +1,32 @@
+All About Repositories: Handling Protocols
+==========================================
+
+A repository is where borg keeps all its backup data. It abstracts the details of
+repositories in order to support multiple remote locations behind a similar interface.
+
+The top-level abstraction is the Repository class which then loads the
+appropriate specific repository class for the location specified by the user.
+
+For example, a ``file://`` location will end up loading a ``LocalRepository`` while
+an ``ssh://`` location will end up loading a ``RemoteRepository`` (which communicates
+with a remote borg instance over ssh).
+
+Adding A New Repository Backend
+-------------------------------
+
+You can see most of what needs to be done by looking at the main ``Repository``
+class in ``repository.py``. Every call it gets, it proxies to a subclass that
+does the real work. That is what you'll write.
+
+A few of the methods are optional and can return ``None`` or do nothing:
+
+- ``get_free_nonce``
+- ``commit_nonce_reservation``
+- ``config`` (if remote)
+- ``save_config()`` (if remote)
+
+Write your new repository class in a file in the ``repositories`` directory.
+
+After writing your new class, add support for it in the ``Repository.__init__``
+method, which inspects a location's protocol and instantiates the appropriate
+backend.

src/borg/archive.py

@@ -44,8 +44,7 @@
 from .patterns import PathPrefixPattern, FnmatchPattern, IECommand
 from .item import Item, ArchiveItem, ItemDiff
 from .platform import acl_get, acl_set, set_flags, get_flags, swidth, hostname
-from .remote import cache_if_remote
-from .repository import Repository, LIST_SCAN_LIMIT
+from .repository import Repository, cache_if_remote
 
 has_lchmod = hasattr(os, 'lchmod')
 

src/borg/archiver.py

@@ -77,8 +77,7 @@
     from .patterns import PatternMatcher
     from .item import Item
     from .platform import get_flags, get_process_id, SyncFile
-    from .remote import RepositoryServer, RemoteRepository, cache_if_remote
-    from .repository import Repository, LIST_SCAN_LIMIT, TAG_PUT, TAG_DELETE, TAG_COMMIT
+    from .repository import Repository, TAG_PUT, TAG_DELETE, TAG_COMMIT, cache_if_remote
     from .selftest import selftest
     from .upgrader import AtticRepositoryUpgrader, BorgRepositoryUpgrader
 except BaseException:
@@ -107,7 +106,7 @@ def argument(args, str_or_bool):
 
 def with_repository(fake=False, invert_fake=False, create=False, lock=True,
                     exclusive=False, manifest=True, cache=False, secure=True,
-                    compatibility=None):
+                    compatibility=None, local_only=False):
     """
     Method decorator for subcommand-handling methods: do_XYZ(self, args, repository, …)
 
@@ -137,19 +136,17 @@ def decorator(method):
         @functools.wraps(method)
         def wrapper(self, args, **kwargs):
             location = args.location  # note: 'location' must be always present in args
-            append_only = getattr(args, 'append_only', False)
-            storage_quota = getattr(args, 'storage_quota', None)
-            make_parent_dirs = getattr(args, 'make_parent_dirs', False)
             if argument(args, fake) ^ invert_fake:
                 return method(self, args, repository=None, **kwargs)
-            elif location.proto == 'ssh':
-                repository = RemoteRepository(location, create=create, exclusive=argument(args, exclusive),
-                                              lock_wait=self.lock_wait, lock=lock, append_only=append_only,
-                                              make_parent_dirs=make_parent_dirs, args=args)
-            else:
-                repository = Repository(location.path, create=create, exclusive=argument(args, exclusive),
-                                        lock_wait=self.lock_wait, lock=lock, append_only=append_only,
-                                        storage_quota=storage_quota, make_parent_dirs=make_parent_dirs)
+
+            repository = Repository(location, create=create,
+                                    exclusive=argument(args, exclusive),
+                                    lock_wait=self.lock_wait, lock=lock,
+                                    args=args)
+
+            if local_only and repository.remote:
+                raise argparse.ArgumentTypeError('"%s": Repository must be local' % location.canonical_path())
+
             with repository:
                 if manifest or cache:
                     kwargs['manifest'], kwargs['key'] = Manifest.load(repository, compatibility)
@@ -237,6 +234,7 @@ def build_matcher(inclexcl_patterns, include_paths):
 
     def do_serve(self, args):
         """Start in server mode. This command is usually not used manually."""
+        from .repositories.remote import RepositoryServer
         RepositoryServer(
             restrict_to_paths=args.restrict_to_paths,
             restrict_to_repositories=args.restrict_to_repositories,
@@ -1618,7 +1616,7 @@ def do_compact(self, args, repository):
         repository.commit(compact=True, cleanup_commits=args.cleanup_commits)
         return EXIT_SUCCESS
 
-    @with_repository(exclusive=True, manifest=False)
+    @with_repository(exclusive=True, manifest=False, local_only=True)
     def do_config(self, args, repository):
         """get, set, and delete values in a repository or cache config file"""
 
@@ -1669,8 +1667,8 @@ def list_config(config):
                 'segments_per_dir': str(DEFAULT_SEGMENTS_PER_DIR),
                 'max_segment_size': str(MAX_SEGMENT_SIZE_LIMIT),
                 'additional_free_space': '0',
-                'storage_quota': repository.storage_quota,
-                'append_only': repository.append_only
+                'storage_quota': '0',
+                'append_only': 'False'
             }
             print('[repository]')
             for key in ['version', 'segments_per_dir', 'max_segment_size',
@@ -1707,7 +1705,7 @@ def list_config(config):
                 validate = cache_validate
             else:
                 config = repository.config
-                save = lambda: repository.save_config(repository.path, repository.config)  # noqa
+                save = lambda: repository.save_config()  # noqa
                 validate = repo_validate
 
             if args.delete:
@@ -2855,7 +2853,7 @@ def define_archive_filters_group(subparser, *, sort_by=True, first_last=True):
                                help='list the configuration of the repo')
 
         subparser.add_argument('location', metavar='REPOSITORY', nargs='?', default='',
-                               type=location_validator(archive=False, proto='file'),
+                               type=location_validator(archive=False),
                                help='repository to configure')
         subparser.add_argument('name', metavar='NAME', nargs='?',
                                help='name of config key')
@@ -4450,21 +4448,10 @@ def main():  # pragma: no cover
                 exit_code = archiver.run(args)
         except Error as e:
             msg = e.get_message()
-            msgid = type(e).__qualname__
+            msgid = e.get_msgid()
             tb_log_level = logging.ERROR if e.traceback else logging.DEBUG
-            tb = "%s\n%s" % (traceback.format_exc(), sysinfo())
+            tb = "%s\n%s" % (e.format_exc(), sysinfo())
             exit_code = e.exit_code
-        except RemoteRepository.RPCError as e:
-            important = e.exception_class not in ('LockTimeout', ) and e.traceback
-            msgid = e.exception_class
-            tb_log_level = logging.ERROR if important else logging.DEBUG
-            if important:
-                msg = e.exception_full
-            else:
-                msg = e.get_message()
-            tb = '\n'.join('Borg server: ' + l for l in e.sysinfo.splitlines())
-            tb += "\n" + sysinfo()
-            exit_code = EXIT_ERROR
         except Exception:
             msg = 'Local Exception'
             msgid = 'Exception'

src/borg/cache.py

@@ -10,7 +10,7 @@
 
 logger = create_logger()
 
-from .constants import CACHE_README, DEFAULT_FILES_CACHE_MODE
+from .constants import CACHE_README, DEFAULT_FILES_CACHE_MODE, LIST_SCAN_LIMIT
 from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer
 from .helpers import Location
 from .helpers import Error
@@ -30,8 +30,7 @@
 from .crypto.file_integrity import IntegrityCheckedFile, DetachedIntegrityCheckedFile, FileIntegrityError
 from .locking import Lock
 from .platform import SaveFile
-from .remote import cache_if_remote
-from .repository import LIST_SCAN_LIMIT
+from .repository import cache_if_remote
 
 # note: cmtime might me either a ctime or a mtime timestamp
 FileCacheEntry = namedtuple('FileCacheEntry', 'age inode size cmtime chunk_ids')

src/borg/crypto/key.py

@@ -231,7 +231,7 @@ def unpack_and_verify_manifest(self, data, force_tam_not_required=False):
         unpacked = unpacker.unpack()
         if b'tam' not in unpacked:
             if tam_required:
-                raise TAMRequiredError(self.repository._location.canonical_path())
+                raise TAMRequiredError(self.repository.location.canonical_path())
             else:
                 logger.debug('TAM not found and not required')
                 return unpacked, False
@@ -549,7 +549,7 @@ def create(cls, repository, args):
 
     @classmethod
     def detect(cls, repository, manifest_data):
-        prompt = 'Enter passphrase for %s: ' % repository._location.canonical_path()
+        prompt = 'Enter passphrase for %s: ' % repository.location.canonical_path()
         key = cls(repository)
         passphrase = Passphrase.env_passphrase()
         if passphrase is None:
@@ -706,9 +706,9 @@ def sanity_check(self, filename, id):
             # we do the magic / id check in binary mode to avoid stumbling over
             # decoding errors if somebody has binary files in the keys dir for some reason.
             if fd.read(len(file_id)) != file_id:
-                raise KeyfileInvalidError(self.repository._location.canonical_path(), filename)
+                raise KeyfileInvalidError(self.repository.location.canonical_path(), filename)
             if fd.read(len(repo_id)) != repo_id:
-                raise KeyfileMismatchError(self.repository._location.canonical_path(), filename)
+                raise KeyfileMismatchError(self.repository.location.canonical_path(), filename)
             return filename
 
     def find_key(self):
@@ -723,7 +723,7 @@ def find_key(self):
                 return self.sanity_check(filename, id)
             except (KeyfileInvalidError, KeyfileMismatchError):
                 pass
-        raise KeyfileNotFoundError(self.repository._location.canonical_path(), get_keys_dir())
+        raise KeyfileNotFoundError(self.repository.location.canonical_path(), get_keys_dir())
 
     def get_new_target(self, args):
         keyfile = os.environ.get('BORG_KEY_FILE')
@@ -761,7 +761,7 @@ class RepoKey(ID_HMAC_SHA_256, KeyfileKeyBase):
     STORAGE = KeyBlobStorage.REPO
 
     def find_key(self):
-        loc = self.repository._location.canonical_path()
+        loc = self.repository.location.canonical_path()
         try:
             self.repository.load_key()
             return loc

src/borg/crypto/nonces.py

@@ -5,7 +5,6 @@
 from ..helpers import get_security_dir
 from ..helpers import bin_to_hex
 from ..platform import SaveFile
-from ..remote import InvalidRPCMethod
 
 from .low_level import bytes_to_long, long_to_bytes
 
@@ -34,14 +33,7 @@ def commit_local_nonce_reservation(self, next_unreserved, start_nonce):
             fd.write(bin_to_hex(long_to_bytes(next_unreserved)))
 
     def get_repo_free_nonce(self):
-        try:
-            return self.repository.get_free_nonce()
-        except InvalidRPCMethod:
-            # old server version, suppress further calls
-            sys.stderr.write("Please upgrade to borg version 1.1+ on the server for safer AES-CTR nonce handling.\n")
-            self.get_repo_free_nonce = lambda: None
-            self.commit_repo_nonce_reservation = lambda next_unreserved, start_nonce: None
-            return None
+        return self.repository.get_free_nonce()
 
     def commit_repo_nonce_reservation(self, next_unreserved, start_nonce):
         self.repository.commit_nonce_reservation(next_unreserved, start_nonce)

src/borg/fuse.py

@@ -23,7 +23,6 @@
 from .helpers import msgpack
 from .item import Item
 from .lrucache import LRUCache
-from .remote import RemoteRepository
 
 # Does this version of llfuse support ns precision?
 have_fuse_xtime_ns = hasattr(llfuse.EntryAttributes, 'st_mtime_ns')
@@ -504,9 +503,8 @@ def pop_option(options, key, present, not_present, wanted_type, int_base=0):
         llfuse.init(self, mountpoint, options)
         if not foreground:
             old_id, new_id = daemonize()
-            if not isinstance(self.repository_uncached, RemoteRepository):
-                # local repo and the locking process' PID just changed, migrate it:
-                self.repository_uncached.migrate_lock(old_id, new_id)
+            # local repo and the locking process' PID just changed, migrate it:
+            self.repository_uncached.migrate_lock(old_id, new_id)
 
         # If the file system crashes, we do not want to umount because in that
         # case the mountpoint suddenly appears to become empty. This can have

src/borg/helpers/errors.py

@@ -1,3 +1,5 @@
+import traceback
+
 from ..constants import *  # NOQA
 
 import borg.crypto.low_level
@@ -20,6 +22,12 @@ def __init__(self, *args):
     def get_message(self):
         return type(self).__doc__.format(*self.args)
 
+    def get_msgid(self):
+        return type(self).__qualname__
+
+    def format_exc(self):
+        return traceback.format_exc()
+
     __str__ = get_message
 
 

src/borg/helpers/parseformat.py

@@ -477,7 +477,7 @@ def canonical_path(self):
                                            path)
 
 
-def location_validator(archive=None, proto=None):
+def location_validator(archive=None):
     def validator(text):
         try:
             loc = Location(text)
@@ -487,11 +487,6 @@ def validator(text):
             raise argparse.ArgumentTypeError('"%s": No archive specified' % text)
         elif archive is False and loc.archive:
             raise argparse.ArgumentTypeError('"%s": No archive can be specified' % text)
-        if proto is not None and loc.proto != proto:
-            if proto == 'file':
-                raise argparse.ArgumentTypeError('"%s": Repository must be local' % text)
-            else:
-                raise argparse.ArgumentTypeError('"%s": Repository must be remote' % text)
         return loc
     return validator
 
@@ -921,13 +916,12 @@ def ellipsis_truncate(msg, space):
 class BorgJsonEncoder(json.JSONEncoder):
     def default(self, o):
         from ..repository import Repository
-        from ..remote import RemoteRepository
         from ..archive import Archive
         from ..cache import LocalCache, AdHocCache
-        if isinstance(o, Repository) or isinstance(o, RemoteRepository):
+        if isinstance(o, Repository):
             return {
-                'id': bin_to_hex(o.id),
-                'location': o._location.canonical_path(),
+                'id': o.id_str,
+                'location': o.location.canonical_path(),
             }
         if isinstance(o, Archive):
             return o.info()