[v2] Enable CRT S3 transfer client by default on 5th gen instance families

.changes/next-release/enhancement-crt-68780.json

@@ -0,0 +1,5 @@
+{
+  "type": "enhancement",
+  "category": "crt",
+  "description": "Enable the CRT S3 transfer client by default on the following EC2 instance families - c5n, c5ad, c5a, c5d, c5, m5zn, m5dn, m5n, m5ad, m5a, m5d, m5, r5dn, r5n, r5ad, r5a, r5d, r5."
+}

awscli/customizations/s3/factory.py

@@ -137,6 +137,24 @@
         'r6a',
         'r6gd',
         'r6g',
+        "c5n",
+        "c5ad",
+        "c5a",
+        "c5d",
+        "c5",
+        "m5zn",
+        "m5dn",
+        "m5n",
+        "m5ad",
+        "m5a",
+        "m5d",
+        "m5",
+        "r5dn",
+        "r5n",
+        "r5ad",
+        "r5a",
+        "r5d",
+        "r5",
     ]
 )
 

awscli/testutils.py

@@ -417,12 +417,23 @@ def setUp(self):
         # future, but for now we just grab that value out of the real
         # os.environ so the patched os.environ has this data and
         # the CLI works.
+        # Pin the S3 transfer client to 'classic'.
+        # Tests that need the CRT client should use BaseCRTTransferClientTest
+        # or explicitly configure preferred_transfer_client = crt.
+        self._config_dir = tempfile.mkdtemp()
+        config_path = os.path.join(self._config_dir, 'config')
+        with open(config_path, 'w') as f:
+            f.write(
+                '[default]\n'
+                's3 =\n'
+                '  preferred_transfer_client = classic\n'
+            )
         self.environ = {
             'AWS_DATA_PATH': os.environ['AWS_DATA_PATH'],
             'AWS_DEFAULT_REGION': 'us-east-1',
             'AWS_ACCESS_KEY_ID': 'access_key',
             'AWS_SECRET_ACCESS_KEY': 'secret_key',
-            'AWS_CONFIG_FILE': '',
+            'AWS_CONFIG_FILE': config_path,
             'AWS_SHARED_CREDENTIALS_FILE': '',
         }
         if os.environ.get('ComSpec'):
@@ -450,6 +461,7 @@ def tearDown(self):
         if self.make_request_is_patched:
             self.make_request_patch.stop()
             self.make_request_is_patched = False
+        shutil.rmtree(self._config_dir, ignore_errors=True)
 
     def before_call(self, params, **kwargs):
         self._store_params(params)

awscli/topics/s3-config.rst

@@ -346,15 +346,18 @@ files to and from S3. Valid choices are:
       ``p5.48xlarge``, ``p5e.48xlarge``, ``p5en.48xlarge``, ``p6-b200.48xlarge``,
       ``p6-b300.48xlarge``, ``trn1.32xlarge``, and ``trn1n.32xlarge``.
 
-    * Of one of these instance families: ``c6a``, ``c6g``, ``c6gd``, ``c6gn``,
-      ``c6i``, ``c6id``, ``c6in``, ``c7a``, ``c7g``, ``c7gd``, ``c7gn``,
+    * Of one of these instance families: ``c5``, ``c5a``, ``c5ad``, ``c5d``,
+      ``c5n``, ``c6a``, ``c6g``, ``c6gd``, ``c6gn``, ``c6i``, ``c6id``,
+      ``c6in``, ``c7a``, ``c7g``, ``c7gd``, ``c7gn``,
       ``c7i``, ``c7i-flex``, ``c8a``, ``c8g``, ``c8gb``, ``c8gd``, ``c8gn``,
       ``c8i``, ``c8i-flex``, ``c8ib``, ``c8id``, ``c8in``, ``c8ine``, ``c9g``,
       ``c9gd``, ``d3``, ``d3en``, ``dl1``, ``g4dn``, ``g5``, ``g5g``, ``g6``,
       ``g6e``, ``h1``, ``i3``, ``i3en``, ``i4i``, ``im4gn``, ``inf1``,
-      ``inf2``, ``is4gen``, ``m6a``, ``m6g``, ``m6gd``, ``m6i``, ``m6id``,
+      ``inf2``, ``is4gen``, ``m5``, ``m5a``, ``m5ad``, ``m5d``, ``m5dn``,
+      ``m5n``, ``m5zn``, ``m6a``, ``m6g``, ``m6gd``, ``m6i``, ``m6id``,
       ``m6idn``, ``m6in``, ``m7a``, ``m7g``, ``m7i``, ``m7i-flex``, ``m8a``,
-      ``m8g``, ``r6a``, ``r6g``, ``r6gd``, ``r6i``, ``r6id``, ``r6idn``,
+      ``m8g``, ``r5``, ``r5a``, ``r5ad``, ``r5d``, ``r5dn``, ``r5n``, ``r6a``,
+      ``r6g``, ``r6gd``, ``r6i``, ``r6id``, ``r6idn``,
       ``r6in``, ``r7a``, ``r7g``, ``r7i``, ``r7iz``, ``x1``, ``x1e``,
       ``x2idn``, ``x2iedn``, and ``x2iezn``.
 

tests/functional/s3/__init__.py

@@ -36,7 +36,12 @@ def setUp(self):
 
     def init_clidriver(self):
         with temporary_file('w') as f:
-            f.write('[default]\n' 's3 =\n' '  max_concurrent_requests = 1\n')
+            f.write(
+                '[default]\n'
+                's3 =\n'
+                '  preferred_transfer_client = classic\n'
+                '  max_concurrent_requests = 1\n'
+            )
             f.flush()
             self.environ['AWS_CONFIG_FILE'] = f.name
             self.driver = create_clidriver()

tests/functional/test_useragent.py

@@ -1,3 +1,5 @@
+import os
+
 from awscli import __version__ as awscli_version
 from awscli.testutils import mock
 from tests import CLIRunner, HTTPResponse
@@ -46,7 +48,10 @@ def test_user_agent_reports_s3_transfer_client(tmp_path):
     upload.write_text('contents')
     cli_runner = CLIRunner()
     cli_runner.add_response(HTTPResponse(headers={'ETag': '"etag"'}))
-    with mock.patch('awscrt.s3.is_optimized_for_system', return_value=False):
+    with mock.patch('awscrt.s3.is_optimized_for_system', return_value=False), \
+         mock.patch('awscrt.s3.get_ec2_instance_type', return_value=None), \
+         mock.patch.dict(os.environ, {}, clear=False) as env:
+        env.pop('AWS_CLI_AUTO_RESOLVE_CLIENT', None)
         result = cli_runner.run(['s3', 'cp', str(upload), 's3://bucket/key'])
     assert result.rc == 0
     ua_string = result.aws_requests[0].http_requests[0].headers['User-Agent']

tests/unit/customizations/s3/test_factory.py

@@ -973,7 +973,7 @@ def test_resolves_to_classic_when_not_on_ec2(
     def test_resolves_to_classic_for_instance_family_not_rolled_out(
         self, resolve_client_type, mock_crt_get_ec2_instance_type
     ):
-        mock_crt_get_ec2_instance_type.return_value = 'm5.2xlarge'
+        mock_crt_get_ec2_instance_type.return_value = 't4g.2xlarge'
         assert resolve_client_type() == constants.CLASSIC_TRANSFER_CLIENT
 
     def test_instance_family_must_match_in_full(
@@ -1217,8 +1217,6 @@ def test_does_not_warn_across_clients(self, warn_unsupported_settings):
     'crt_is_optimized_for_system,crt_running_in_other_process,'
     'expected_transfer_manager_cls',
     [
-        (None, {}, False, False, TransferManager),
-        ('auto', {}, False, False, TransferManager),
         ('classic', {}, False, False, TransferManager),
         ('crt', {}, False, False, CRTTransferManager),
         # "default" is a supported alias for "classic"
@@ -1239,8 +1237,6 @@ def test_does_not_warn_across_clients(self, warn_unsupported_settings):
         ('classic', {'paths_type': 's3s3'}, True, False, TransferManager),
         ('crt', {'paths_type': 's3s3'}, True, False, TransferManager),
         # Streaming operations use requested transfer client
-        (None, {'is_stream': True}, False, False, TransferManager),
-        ('auto', {'is_stream': True}, False, False, TransferManager),
         ('classic', {'is_stream': True}, False, False, TransferManager),
         ('crt', {'is_stream': True}, False, False, CRTTransferManager),
     ],
@@ -1298,7 +1294,6 @@ def test_factory_always_acquires_crt_transfer_lock_for_crt_manager(
 @pytest.mark.parametrize(
     'preferred_transfer_client,crt_is_optimized_for_system',
     [
-        ('auto', False),
         ('classic', False),
         ('classic', True),
     ],
@@ -1342,8 +1337,6 @@ def _create_transfer_manager_from_factory(
     [
         ('classic', {}, False, 'Ad'),
         ('crt', {}, False, 'Ae'),
-        (None, {}, False, 'Af'),
-        ('auto', {}, False, 'Af'),
         (None, {}, True, 'Ag'),
         ('auto', {}, True, 'Ag'),
         # S3 copies always use the classic client.