I'm trying to run a test that was previously working but has suddenly stopped running but now i seem to get an error on all my tests e.g.
from httmock import HTTMock
from unittest import TestCase
from unittest.mock import patch, call, mock_open, MagicMock, Mock, ANY
import os.path
import os
from src.operators import InjestDictDescriptionOperator
from airflow.hooks.base_hook import BaseHook
from airflow.hooks.postgres_hook import PostgresHook
from airflow.hooks.S3_hook import S3Hook
class TestInjestDictDescriptionOperator(TestCase):
def setUp(self):
# hook patches
self.open_file_mock = patch('builtins.open').start()
self.os_path_isdir = patch.object(os.path, 'isdir').start()
self.os_makedirs = patch.object(os, 'makedirs').start()
self.open_file_write_mock = self.open_file_mock.return_value.__enter__.return_value.write
# prepare the target
self.target = InjestDictDescriptionOperator(
task_id='InjestDictDescriptionOperatorTest',
sql=None,
postgres_conn_id='test',
aws_conn_id='s3-conn-1',
s3_bucket_name=‘data’,
output_path='output/path/1')
def tearDown(self):
patch.stopall()
def testTmpFolderCreationIfItDoesntExist(self):
self.os_path_isdir.return_value = False
self.target.execute(None)
self.os_makedirs.assert_called_with('/tmp/')
def testTmpFolderNotCreatedIfItExists(self):
self.os_path_isdir.return_value = True
self.target.execute(None)
self.os_makedirs.assert_not_called()
def testTmpFileCreation(self):
self.target.execute(None)
self.open_file_mock.assert_called_with(
'/tmp/modelling/temp.txt',
'w+',
encoding='utf-8')
def testTmpFileDataDump(self):
self.target.execute(None)
self.open_file_write_mock.assert_has_calls(
[ call(f"{doc['name']}\n") for doc in self.dummy_data ]
, any_order=False)
The traceback details of the problem are
ests/operators/modelling/language/test_injest_dict_description_operator.py:9: in <module>
from src.operators.modelling.language import InjestDictDescriptionOperator
src/operators/modelling/language/__init__.py:1: in <module>
from .injest_onboarded_commands_operator import Operator as InjestOnboardedCommandsOperator
src/operators/modelling/language/injest_onboarded_commands_operator.py:9: in <module>
from airflow.models import BaseOperator
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/__init__.py:50: in <module>
from airflow.models import DAG # noqa: E402
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/__init__.py:21: in <module>
from airflow.models.baseoperator import BaseOperator, BaseOperatorLink # noqa: F401
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/baseoperator.py:43: in <module>
from airflow.models.dag import DAG
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dag.py:52: in <module>
from airflow.models.dagbag import DagBag
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:50: in <module>
class DagBag(BaseDagBag, LoggingMixin):
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:80: in DagBag
DAGBAG_IMPORT_TIMEOUT = conf.getint('core', 'DAGBAG_IMPORT_TIMEOUT')
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/configuration.py:414: in getint
return int(self.get(section, key, **kwargs))
E ValueError: invalid literal for int() with base 10: '30.0'collection failure
tests/operators/modelling/language/test_injest_dict_description_operator.py:9: in <module>
from src.operators.modelling.language import InjestDictDescriptionOperator
src/operators/modelling/language/__init__.py:1: in <module>
from .injest_onboarded_commands_operator import Operator as InjestOnboardedCommandsOperator
src/operators/modelling/language/injest_onboarded_commands_operator.py:9: in <module>
from airflow.models import BaseOperator
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/__init__.py:50: in <module>
from airflow.models import DAG # noqa: E402
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/__init__.py:21: in <module>
from airflow.models.baseoperator import BaseOperator, BaseOperatorLink # noqa: F401
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/baseoperator.py:43: in <module>
from airflow.models.dag import DAG
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dag.py:52: in <module>
from airflow.models.dagbag import DagBag
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:50: in <module>
class DagBag(BaseDagBag, LoggingMixin):
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/models/dagbag.py:80: in DagBag
DAGBAG_IMPORT_TIMEOUT = conf.getint('core', 'DAGBAG_IMPORT_TIMEOUT')
../../../../.pyenv/versions/3.6.10/lib/python3.6/site-packages/airflow/configuration.py:414: in getint
return int(self.get(section, key, **kwargs))
E ValueError: invalid literal for int() with base 10: '30.0'
The only thing that changed was apache-airflow. I upgraded to the latest version 2.0 but then realised i would need to refactor parts of my code so downgraded to a later version.