Merge pull request #324 from cpnota/develop

v0.9.1

.github/workflows/python-package.yml

@@ -15,7 +15,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.8, 3.9]
+        python-version: [3.8, 3.11]
 
     steps:
     - uses: actions/checkout@v2
@@ -25,9 +25,8 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
       run: |
-        sudo apt-get install swig
-        sudo apt-get install unrar
-        pip install torch~=1.11 --extra-index-url https://download.pytorch.org/whl/cpu
+        python -m pip install --upgrade pip
+        pip install torch~=2.0 --extra-index-url https://download.pytorch.org/whl/cpu
         make install
     - name: Lint code
       run: |

.github/workflows/python-publish.yml

@@ -1,33 +1,34 @@
 # This workflow will upload a Python Package using Twine when a release is created
-# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries
+# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries
 
 name: Upload Python Package
 
 on:
   release:
-    types: [created]
+    types: [published]
+
+permissions:
+  contents: read
 
 jobs:
   deploy:
-
     runs-on: ubuntu-latest
-
-    environment: deployment
-
+    environment: publish
+    permissions:
+      id-token: write
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v3
       with:
-        python-version: '3.x'
+        python-version: 3.11
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install setuptools wheel twine
-    - name: Build and publish
-      env:
-        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-      run: |
-        python setup.py sdist bdist_wheel
-        twine upload dist/*
+        pip install torch~=2.0 --extra-index-url https://download.pytorch.org/whl/cpu
+        pip install setuptools wheel
+        make install
+    - name: Build package
+      run: make build
+    - name: Publish package
+      uses: pypa/gh-action-pypi-publish@release/v1

.readthedocs.yml

@@ -1,26 +1,16 @@
-# .readthedocs.yml
-# Read the Docs configuration file
-# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
-
-# Required
 version: 2
 
-# Build documentation in the docs/ directory with Sphinx
-sphinx:
-  configuration: docs/source/conf.py
-
-# Build documentation with MkDocs
-#mkdocs:
-#  configuration: mkdocs.yml
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.11"
 
-# Optionally build your docs in additional formats such as PDF and ePub
-formats: all
-
-# Optionally set the version of Python and requirements required to build your docs
 python:
-  version: 3.7
   install:
     - method: pip
       path: .
       extra_requirements:
         - docs
+
+sphinx:
+  configuration: docs/source/conf.py

Makefile

@@ -11,10 +11,13 @@ integration-test:
 	python -m unittest discover -s integration -p "*test.py"
 
 lint:
-	flake8 --ignore "E501,E731,E74,E402,F401,W503,E128" all
+	black --check all benchmarks examples integration setup.py
+	isort --profile black --check all benchmarks examples integration setup.py
+	flake8 --select "F401" all benchmarks examples integration setup.py
 
 format:
-	autopep8 --in-place --aggressive --aggressive --ignore "E501,E731,E74,E402,F401,W503,E128" -r all
+	black all benchmarks examples integration setup.py
+	isort --profile black all benchmarks examples integration setup.py
 
 tensorboard:
 	tensorboard --logdir runs

README.md

@@ -21,10 +21,11 @@ Additionally, we provide an [example project](https://github.com/cpnota/all-exam
 
 ## High-Quality Reference Implementations
 
-The `autonomous-learning-library` separates reinforcement learning agents into two modules: `all.agents`, which provides flexible, high-level implementations of many common algorithms which can be adapted to new problems and environments, and `all.presets` which provides specific instansiations of these agents tuned for particular sets of environments, including Atari games, classic control tasks, and PyBullet robotics simulations. Some benchmark results showing results on-par with published results can be found below:
+The `autonomous-learning-library` separates reinforcement learning agents into two modules: `all.agents`, which provides flexible, high-level implementations of many common algorithms which can be adapted to new problems and environments, and `all.presets` which provides specific instansiations of these agents tuned for particular sets of environments, including Atari games, classic control tasks, and MuJoCo/Pybullet robotics simulations. Some benchmark results showing results on-par with published results can be found below:
 
-![atari40](benchmarks/atari40.png)
-![pybullet](benchmarks/pybullet.png)
+![atari40](benchmarks/atari_40m.png)
+![atari40](benchmarks/mujoco_v4.png)
+![pybullet](benchmarks/pybullet_v0.png)
 
 As of today, `all` contains implementations of the following deep RL algorithms:
 

all/__init__.py

@@ -1,26 +1,16 @@
-import all.agents
-import all.approximation
-import all.core
-import all.environments
-import all.logging
-import all.memory
-import all.nn
-import all.optim
-import all.policies
-import all.presets
 from all.core import State, StateArray
 
 __all__ = [
-    'agents',
-    'approximation',
-    'core',
-    'environments',
-    'logging',
-    'memory',
-    'nn',
-    'optim',
-    'policies',
-    'presets',
-    'State',
-    'StateArray'
+    "agents",
+    "approximation",
+    "core",
+    "environments",
+    "logging",
+    "memory",
+    "nn",
+    "optim",
+    "policies",
+    "presets",
+    "State",
+    "StateArray",
 ]

all/agents/__init__.py

@@ -15,7 +15,6 @@
 from .vqn import VQN, VQNTestAgent
 from .vsarsa import VSarsa, VSarsaTestAgent
 
-
 __all__ = [
     # Agent interfaces
     "Agent",

all/agents/_agent.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+
 from all.optim import Schedulable
 
 

all/agents/_multiagent.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+
 from all.optim import Schedulable
 
 

all/agents/_parallel_agent.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+
 from all.optim import Schedulable
 
 

all/agents/a2c.py

@@ -1,7 +1,8 @@
-import torch
 from torch.nn.functional import mse_loss
+
 from all.logging import DummyLogger
 from all.memory import NStepAdvantageBuffer
+
 from ._agent import Agent
 from ._parallel_agent import ParallelAgent
 
@@ -28,15 +29,15 @@ class A2C(ParallelAgent):
     """
 
     def __init__(
-            self,
-            features,
-            v,
-            policy,
-            discount_factor=0.99,
-            entropy_loss_scaling=0.01,
-            n_envs=None,
-            n_steps=4,
-            logger=DummyLogger()
+        self,
+        features,
+        v,
+        policy,
+        discount_factor=0.99,
+        entropy_loss_scaling=0.01,
+        n_envs=None,
+        n_steps=4,
+        logger=DummyLogger(),
     ):
         if n_envs is None:
             raise RuntimeError("Must specify n_envs.")
@@ -80,7 +81,9 @@ def _train(self, next_states):
             value_loss = mse_loss(values, targets)
             policy_gradient_loss = -(distribution.log_prob(actions) * advantages).mean()
             entropy_loss = -distribution.entropy().mean()
-            policy_loss = policy_gradient_loss + self.entropy_loss_scaling * entropy_loss
+            policy_loss = (
+                policy_gradient_loss + self.entropy_loss_scaling * entropy_loss
+            )
             loss = value_loss + policy_loss
 
             # backward pass
@@ -90,16 +93,16 @@ def _train(self, next_states):
             self.features.step()
 
             # record metrics
-            self.logger.add_info('entropy', -entropy_loss)
-            self.logger.add_info('normalized_value_error', value_loss / targets.var())
+            self.logger.add_info("entropy", -entropy_loss)
+            self.logger.add_info("normalized_value_error", value_loss / targets.var())
 
     def _make_buffer(self):
         return NStepAdvantageBuffer(
             self.v,
             self.features,
             self.n_steps,
             self.n_envs,
-            discount_factor=self.discount_factor
+            discount_factor=self.discount_factor,
         )
 
 

all/agents/c51.py

@@ -1,6 +1,8 @@
-import torch
 import numpy as np
+import torch
+
 from all.logging import DummyLogger
+
 from ._agent import Agent
 
 
@@ -26,16 +28,16 @@ class C51(Agent):
     """
 
     def __init__(
-            self,
-            q_dist,
-            replay_buffer,
-            discount_factor=0.99,
-            eps=1e-5,
-            exploration=0.02,
-            minibatch_size=32,
-            replay_start_size=5000,
-            update_frequency=1,
-            logger=DummyLogger(),
+        self,
+        q_dist,
+        replay_buffer,
+        discount_factor=0.99,
+        eps=1e-5,
+        exploration=0.02,
+        minibatch_size=32,
+        replay_start_size=5000,
+        update_frequency=1,
+        logger=DummyLogger(),
     ):
         # objects
         self.q_dist = q_dist
@@ -81,7 +83,9 @@ def _best_actions(self, probs):
     def _train(self):
         if self._should_train():
             # sample transitions from buffer
-            states, actions, rewards, next_states, weights = self.replay_buffer.sample(self.minibatch_size)
+            states, actions, rewards, next_states, weights = self.replay_buffer.sample(
+                self.minibatch_size
+            )
             # forward pass
             dist = self.q_dist(states, actions)
             # compute target distribution
@@ -100,14 +104,15 @@ def _train(self):
 
     def _should_train(self):
         self._frames_seen += 1
-        return self._frames_seen > self.replay_start_size and self._frames_seen % self.update_frequency == 0
+        return (
+            self._frames_seen > self.replay_start_size
+            and self._frames_seen % self.update_frequency == 0
+        )
 
     def _compute_target_dist(self, states, rewards):
         actions = self._best_actions(self.q_dist.no_grad(states))
         dist = self.q_dist.target(states, actions)
-        shifted_atoms = (
-            rewards.view((-1, 1)) + self.discount_factor * self.q_dist.atoms
-        )
+        shifted_atoms = rewards.view((-1, 1)) + self.discount_factor * self.q_dist.atoms
         return self.q_dist.project(dist, shifted_atoms)
 
     def _kl(self, dist, target_dist):
@@ -117,7 +122,7 @@ def _kl(self, dist, target_dist):
 
 
 class C51TestAgent(Agent):
-    def __init__(self, q_dist, n_actions, exploration=0.):
+    def __init__(self, q_dist, n_actions, exploration=0.0):
         self.q_dist = q_dist
         self.n_actions = n_actions
         self.exploration = exploration