Merge branch 'master' into multi-periods-dataframe

.circleci/config.yml

@@ -2,7 +2,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: python:3.7
+      - image: python:3.9
 
     steps:
       - checkout

.github/has-functional-changes.sh

@@ -1,5 +1,7 @@
 #! /usr/bin/env bash
 
+echo "Run has functionnal changes"
+
 IGNORE_DIFF_ON="README.md CONTRIBUTING.md Makefile .gitignore .circleci/* .github/*"
 
 last_tagged_commit=`git describe --tags --abbrev=0 --first-parent`  # --first-parent ensures we don't follow tags not published in master through an unlikely intermediary merge commit
@@ -8,5 +10,6 @@ if git diff-index --name-only --exit-code $last_tagged_commit -- . `echo " $IGNO
 then
   echo "No functional changes detected."
   exit 1
-else echo "The functional files above were changed."
+else
+  echo "The functional files above were changed."
 fi

.github/workflows/workflow.yml

@@ -12,7 +12,7 @@ jobs:
       fail-fast: true
       matrix:
         os: ["ubuntu-20.04"]  # On peut ajouter "macos-latest" si besoin
-        python-version: ["3.7.9", "3.8.9", "3.9.9"]
+        python-version: ["3.9.9", "3.10.6"]
     steps:
       - name: Checkout
         uses: actions/checkout@v3
@@ -48,13 +48,14 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8.9
+          python-version: 3.9.9
       - name: Cache build
         id: restore-build
         uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}
           key: build-${{ env.pythonLocation }}-${{ hashFiles('setup.py') }}-${{ github.sha }}-ubuntu-20.04
+      - run: pip install -e .[dev]  # Need to install linter
       - run: make check-syntax-errors
       - run: make check-style
 
@@ -65,7 +66,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ "ubuntu-20.04" ]  # On peut ajouter "macos-latest" si besoin
-        python-version: ["3.7.9", "3.8.9", "3.9.9"]
+        python-version: ["3.9.9", "3.10.6"]
     env:
       # Needed for coveralls
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
@@ -97,7 +98,7 @@ jobs:
   #     fail-fast: true
   #     matrix:
   #       os: [ "ubuntu-20.04" ]  # On peut ajouter "macos-latest" si besoin
-  #       python-version: ["3.7.9"]
+  #       python-version: ["3.9.9"]
   #   steps:
   #     - uses: actions/checkout@v3
   #     - name: Set up Python
@@ -123,7 +124,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8.9
+          python-version: 3.9.9
       - name: Check version number has been properly updated
         run: "${GITHUB_WORKSPACE}/.github/is-version-number-acceptable.sh"
 
@@ -143,7 +144,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.8.9
+          python-version: 3.9.9
       - id: stop-early
         run: if "${GITHUB_WORKSPACE}/.github/has-functional-changes.sh" ; then echo "{status}={success}"  >> $GITHUB_STATE ; fi
 
@@ -161,7 +162,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v4
         with:
-          python-version: 3.7.9
+          python-version: 3.9.9
       - name: Cache build
         id: restore-build
         uses: actions/cache@v3
@@ -186,7 +187,7 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          python-version: "3.7.9"
+          python-version: 3.9.9
           # Add conda-forge for OpenFisca-Core
           channels: conda-forge
           activate-environment: true
@@ -212,7 +213,7 @@ jobs:
       - uses: conda-incubator/setup-miniconda@v2
         with:
           auto-update-conda: true
-          python-version: "3.9.9"
+          python-version: 3.9.9
           # Add conda-forge for OpenFisca-Core
           channels: conda-forge
           activate-environment: true

CHANGELOG.md

@@ -1,5 +1,33 @@
 # Changelog
 
+# 1.0.0 [#252](https://github.com/openfisca/openfisca-survey-manager/pull/252)
+
+* Technical improvement.
+- Impacted periods: all.
+- Impacted areas: all.
+- Details:
+  - Upgrade every dependencies & use their latest versions
+
+### 0.47.2 [#247](https://github.com/openfisca/openfisca-survey-manager/pull/247)
+
+* Technical changes
+- Fix `default_config_directory` for use with `openfisca-france-data` in a CI
+
+## 0.47.1 [#246](https://github.com/openfisca/openfisca-survey-manager/pull/246)
+
+* Technical changes
+- Fix `default_config_directory` for use with `openfisca-france-data` in a CI
+
+### 0.46.19 [#244](https://github.com/openfisca/openfisca-survey-manager/pull/244)
+
+* Technical changes
+- Bump to publish package
+
+### 0.46.18 [#243](https://github.com/openfisca/openfisca-survey-manager/pull/243)
+
+* Technical changes
+- Bump to publish package
+
 ### 0.46.17 [#242](https://github.com/openfisca/openfisca-survey-manager/pull/242)
 
 * Technical changes

README.md

@@ -34,9 +34,7 @@ Il fournit deux fonctionnalités principales:
 
 ## Environment
 
-OpenFisca-Survey-Manager runs on Python 3.7. More recent versions should work, but are not tested.
-
-> Backward compatibility with Python 2.7 has been dropped since January 1st, 2019.
+OpenFisca-Survey-Manager runs on Python 3.9. More recent versions should work, but are not tested.
 
 ## Usage
 
@@ -64,7 +62,7 @@ It comes with `build-collection` command that we will use in the next steps.
 
 Create an anvironment and install openfisca-survey-manager
 ```
-conda create -n survey python=3.8
+conda create -n survey python=3.9
 conda activate survey
 conda install -c conda-forge -c openfisca openfisca-survey-manager
 ```

openfisca_survey_manager/__init__.py

@@ -1,13 +1,11 @@
-
-
 import logging
 import os
 import pkg_resources
 
 
 log = logging.getLogger(__name__)
 
-# Hack to for use at the CASD (shared user)
+# Hack for use at the CASD (shared user)
 # Use taxipp/.config/ directory if exists as default_config_files_directory
 try:
     taxipp_location = pkg_resources.get_distribution('taxipp').location
@@ -18,26 +16,41 @@
 if taxipp_location is None or not os.path.exists(default_config_files_directory):
     default_config_files_directory = None
 
-# Travis tests
-if 'CI' in os.environ:
-    is_in_ci = True
-else:
-    is_in_ci = False
 
+# Hack for uising with france-data on a CI or locally
+try:
+    france_data_location = pkg_resources.get_distribution('openfisca-france_data').location
+    from xdg import BaseDirectory
+    default_config_files_directory = BaseDirectory.save_config_path('openfisca-survey-manager')
+except pkg_resources.DistributionNotFound:
+    france_data_location = None
+
+if france_data_location is None or not os.path.exists(default_config_files_directory):
+    default_config_files_directory = None
+
+# Run CI when testing openfisca-survey-manager for example GitHub actions
 test_config_files_directory = os.path.join(
     pkg_resources.get_distribution('openfisca-survey-manager').location,
     'openfisca_survey_manager',
     'tests',
     'data_files',
     )
+
 with open(os.path.join(test_config_files_directory, 'config_template.ini')) as file:
     config_ini = file.read()
 
 config_ini = config_ini.format(location = pkg_resources.get_distribution('openfisca-survey-manager').location)
 with open(os.path.join(test_config_files_directory, 'config.ini'), "w+") as file:
     file.write(config_ini)
 
-if is_in_ci:
+# GitHub actions test
+if 'CI' in os.environ:
+    is_in_ci = True
+else:
+    is_in_ci = False
+
+
+if is_in_ci and default_config_files_directory is None:
     default_config_files_directory = test_config_files_directory
 
 if default_config_files_directory is None:

openfisca_survey_manager/aggregates.py

@@ -116,6 +116,9 @@ def compute_difference(self, target = "baseline", default = 'actual', amount = T
         base_data_frame = self.base_data_frame if self.base_data_frame is not None else self.compute_aggregates()
 
         difference_data_frame = base_data_frame[['label', 'entity']].copy()
+        # Remove duplicates
+        difference_data_frame = difference_data_frame.loc[:, ~difference_data_frame.columns.duplicated()].copy()
+
         quantities = list()
         quantities += ['amount'] if amount else None
         quantities += ['beneficiaries'] if beneficiaries else None
@@ -402,6 +405,9 @@ def get_data_frame(
             'beneficiaries_relative_difference'
             ]
         if difference_data_frame is not None:
+            # Remove eventual duplication
+            difference_data_frame = difference_data_frame.loc[:, ~difference_data_frame.columns.duplicated()].copy()
+            aggregates_data_frame = aggregates_data_frame.loc[:, ~aggregates_data_frame.columns.duplicated()].copy()
             df = aggregates_data_frame.merge(difference_data_frame, how = 'left')[columns]
         else:
             columns = [column for column in columns if column in aggregates_data_frame.columns]

openfisca_survey_manager/coicop.py

@@ -29,7 +29,7 @@ def build_coicop_level_nomenclature(level, keep_code = False, to_csv = False):
             )
     except Exception as e:
         log.info("Error when reading nomenclature coicop source data for level {}".format(level))
-        raise(e)
+        raise e
 
     data_frame.reset_index(inplace = True)
     data_frame.rename(columns = {0: 'code_coicop', 1: 'label_{}'.format(level[:-1])}, inplace = True)

openfisca_survey_manager/config.py

@@ -1,5 +1,3 @@
-
-
 import os
 import configparser
 

openfisca_survey_manager/read_sas.py

@@ -16,6 +16,6 @@ def read_sas(sas_file_path, clean = False) -> DataFrame:
         except ImportError as e2:
             log.info("Neither pyreadstat nor SAS7BDAT are available")
             print(e1)  # noqa analysis:ignore
-            raise(e2)
+            raise e2
 
     return data_frame

openfisca_survey_manager/scenarios.py

@@ -357,7 +357,7 @@ def cast_to_target_entity(simulation):
         marginal_rate = 1 - np.divide(
             numerator,
             denominator,
-            out = np.full_like(numerator, value_for_zero_varying_variable, dtype = np.float),
+            out = np.full_like(numerator, value_for_zero_varying_variable, dtype = np.floating),
             where = (denominator != 0)
             )
 
@@ -692,7 +692,7 @@ def create_data_frame_by_entity(self, variables = None, expressions = None, filt
 
         if index:
             person_data_frame = openfisca_data_frame_by_entity_key.get(person_entity.key)
-            person_data_frame.index.name = self.id_variable_by_entity_key["person"]
+            person_data_frame.index.name = self.id_variable_by_entity_key.get("person", "person_id")
             if person_data_frame is None:
                 person_data_frame = pd.DataFrame()
             for entity in non_person_entities:
@@ -1639,7 +1639,7 @@ def init_variable_in_entity(simulation, entity, variable_name, series, period):
             pd.api.types.is_categorical_dtype(series)
             or not (
                 np.issubdtype(series.values.dtype, np.integer)
-                or np.issubdtype(series.values.dtype, np.float)
+                or np.issubdtype(series.values.dtype, float)
                 )
             )
         )

openfisca_survey_manager/scripts/build_collection.py

@@ -147,8 +147,8 @@ def check_template_config_files(config_files_directory: str):
     config_template_ini_path = os.path.join(config_files_directory, 'config_template.ini')
 
     if os.path.exists(config_files_directory):
-        config_files_do_not_exist = not(os.path.exists(raw_data_ini_path) and os.path.exists(config_ini_path))
-        templates_config_files_do_not_exist = not(
+        config_files_do_not_exist = not (os.path.exists(raw_data_ini_path) and os.path.exists(config_ini_path))
+        templates_config_files_do_not_exist = not (
             os.path.exists(raw_data_template_ini_path) and os.path.exists(config_template_ini_path))
 
         if config_files_do_not_exist:

openfisca_survey_manager/temporary.py

@@ -46,7 +46,7 @@ def func_wrapper(*args, **kwargs):
             try:
                 return func(*args, temporary_store = temporary_store, **kwargs)
             except Exception as e:
-                raise(e)
+                raise e
             finally:
                 gc.collect()
                 if just_openned:

openfisca_survey_manager/tests/test_create_data_frame_by_entity.py

@@ -27,8 +27,8 @@ def test_create_data_frame_by_entity_with_index(self):
             index = True
             )
         for entity, input_dataframe in data_frame_by_entity.items():
-            print(f"{entity} for {period}")
-            print(input_dataframe.columns)
+            print(f"{entity} for {period}")  # noqa T201
+            print(input_dataframe.columns)  # noqa T201
             if entity == "person":
                 self.assertIn("person_id", input_dataframe.columns.to_list())
             if entity == "household":

openfisca_survey_manager/tests/test_scenario.py

@@ -175,14 +175,14 @@ def test_init_from_data(nb_persons = 10, nb_groups = 5, salary_max_value = 50000
         if cols in data_out['person']:
             pass
         else:
-            print('Columns lost in person table: ', cols)
+            print('Columns lost in person table: ', cols)  # noqa T201
     assert data_out['person']['salary'].equals(table_ind['salary'])
 
     for cols in table_men:
         if cols in data_out['household']:
             pass
         else:
-            print('Columns lost in household table: ', cols)
+            print('Columns lost in household table: ', cols)  # noqa T201
     assert data_out['household']['rent'].equals(table_men['rent'])