From 7529c1970c4091057cbc02aa1dd183ec316bd6ad Mon Sep 17 00:00:00 2001 From: Sangram Date: Mon, 10 Jun 2019 23:31:26 +0530 Subject: [PATCH 1/7] Add new funcion to hide the paging, fields and versions arguments --- storage/google/cloud/storage/bucket.py | 50 ++++++++++++++++++++++++++ storage/tests/unit/test_bucket.py | 44 +++++++++++++++++++++++ 2 files changed, 94 insertions(+) diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index f3097a5f7eda..d98f40202835 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -802,6 +802,56 @@ def list_blobs( iterator.bucket = self iterator.prefixes = set() return iterator + + def list_blob_objects( + self, + max_results=None, + prefix=None, + delimiter=None, + projection="noAcl", + client=None, + ): + """Return an iterator used to find blobs in the bucket. + + If :attr:`user_project` is set, bills the API request to that project. + + :type max_results: int + :param max_results: + (Optional) The maximum number of blobs in each page of results + from this request. Non-positive values are ignored. Defaults to + a sensible value set by the API. + + :type prefix: str + :param prefix: (Optional) prefix used to filter blobs. + + :type delimiter: str + :param delimiter: (Optional) Delimiter, used with ``prefix`` to + emulate hierarchy. + + :type projection: str + :param projection: (Optional) If used, must be 'full' or 'noAcl'. + Defaults to ``'noAcl'``. Specifies the set of + properties to return. + + :type client: :class:`~google.cloud.storage.client.Client` + :param client: (Optional) The client to use. If not passed, falls back + to the ``client`` stored on the current bucket. + + :rtype: :class:`~google.api_core.page_iterator.Iterator` + :returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` + in this bucket matching the arguments. + """ + iterator = self.list_blobs( + max_results=max_results, + page_token=None, + prefix=prefix, + delimiter=delimiter, + versions=None, + projection=projection, + fields=None, + client=client, + ) + return iterator def list_notifications(self, client=None): """List Pub / Sub notifications for this bucket. diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index 31b7d293010e..5628a403563e 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -742,6 +742,19 @@ def test_list_blobs_defaults(self): self.assertEqual(kw["path"], "/b/%s/o" % NAME) self.assertEqual(kw["query_params"], {"projection": "noAcl"}) + def test_list_blobs_objects_defaults(self): + NAME = "name" + connection = _Connection({"items": []}) + client = _Client(connection) + bucket = self._make_one(client=client, name=NAME) + iterator = bucket.list_blob_objects() + blobs = list(iterator) + self.assertEqual(blobs, []) + kw, = connection._requested + self.assertEqual(kw["method"], "GET") + self.assertEqual(kw["path"], "/b/%s/o" % NAME) + self.assertEqual(kw["query_params"], {"projection": "noAcl"}) + def test_list_blobs_w_all_arguments_and_user_project(self): NAME = "name" USER_PROJECT = "user-project-123" @@ -782,6 +795,37 @@ def test_list_blobs_w_all_arguments_and_user_project(self): self.assertEqual(kw["path"], "/b/%s/o" % NAME) self.assertEqual(kw["query_params"], EXPECTED) + def test_list_blob_objects_w_all_arguments_and_user_project(self): + NAME = "name" + USER_PROJECT = "user-project-123" + MAX_RESULTS = 10 + PREFIX = "subfolder" + DELIMITER = "/" + PROJECTION = "full" + EXPECTED = { + "maxResults": 10, + "prefix": PREFIX, + "delimiter": DELIMITER, + "projection": PROJECTION, + "userProject": USER_PROJECT, + } + connection = _Connection({"items": []}) + client = _Client(connection) + bucket = self._make_one(name=NAME, user_project=USER_PROJECT) + iterator = bucket.list_blob_objects( + max_results=MAX_RESULTS, + prefix=PREFIX, + delimiter=DELIMITER, + projection=PROJECTION, + client=client, + ) + blobs = list(iterator) + self.assertEqual(blobs, []) + kw, = connection._requested + self.assertEqual(kw["method"], "GET") + self.assertEqual(kw["path"], "/b/%s/o" % NAME) + self.assertEqual(kw["query_params"], EXPECTED) + def test_list_blobs(self): NAME = "name" connection = _Connection({"items": []}) From 2e89bda25bd6f4fee39bd2f3b1368b6c9170fa4f Mon Sep 17 00:00:00 2001 From: Sangram Date: Thu, 13 Jun 2019 20:27:23 +0530 Subject: [PATCH 2/7] Add default fields for item name --- storage/google/cloud/storage/bucket.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index d98f40202835..5b69eb04ab64 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -802,13 +802,14 @@ def list_blobs( iterator.bucket = self iterator.prefixes = set() return iterator - + def list_blob_objects( self, max_results=None, prefix=None, delimiter=None, projection="noAcl", + fields=None, client=None, ): """Return an iterator used to find blobs in the bucket. @@ -833,6 +834,15 @@ def list_blob_objects( Defaults to ``'noAcl'``. Specifies the set of properties to return. + :type fields: str + :param fields: + (Optional) Selector specifying which fields to include + in a partial response. Must be a list of fields. For + example to get a partial response with just the next + page token and the name and language of each blob returned: + ``'items(name,contentLanguage),nextPageToken'``. + See: https://cloud.google.com/storage/docs/json_api/v1/parameters#fields + :type client: :class:`~google.cloud.storage.client.Client` :param client: (Optional) The client to use. If not passed, falls back to the ``client`` stored on the current bucket. @@ -841,14 +851,16 @@ def list_blob_objects( :returns: Iterator of all :class:`~google.cloud.storage.blob.Blob` in this bucket matching the arguments. """ + + if fields is None: + fields="items(name)" + iterator = self.list_blobs( max_results=max_results, - page_token=None, prefix=prefix, delimiter=delimiter, - versions=None, projection=projection, - fields=None, + fields=fields, client=client, ) return iterator From 76d575c78c06293388e9b0b8f321f36271a73f4d Mon Sep 17 00:00:00 2001 From: Sangram Date: Tue, 18 Jun 2019 00:36:27 +0530 Subject: [PATCH 3/7] Fix blacken --- storage/google/cloud/storage/bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index 5b69eb04ab64..ebf7ecc6de2a 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -853,7 +853,7 @@ def list_blob_objects( """ if fields is None: - fields="items(name)" + fields = "items(name)" iterator = self.list_blobs( max_results=max_results, From b4a0da0402bc376a9c81f21a2ae3b3f0619bc6a6 Mon Sep 17 00:00:00 2001 From: Sangram Date: Thu, 20 Jun 2019 03:45:49 +0530 Subject: [PATCH 4/7] Code review changes --- storage/google/cloud/storage/bucket.py | 3 --- storage/tests/unit/test_bucket.py | 3 +++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/storage/google/cloud/storage/bucket.py b/storage/google/cloud/storage/bucket.py index ebf7ecc6de2a..13913b9cdc21 100644 --- a/storage/google/cloud/storage/bucket.py +++ b/storage/google/cloud/storage/bucket.py @@ -852,9 +852,6 @@ def list_blob_objects( in this bucket matching the arguments. """ - if fields is None: - fields = "items(name)" - iterator = self.list_blobs( max_results=max_results, prefix=prefix, diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index 5628a403563e..84e47b6c6fe7 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -802,11 +802,13 @@ def test_list_blob_objects_w_all_arguments_and_user_project(self): PREFIX = "subfolder" DELIMITER = "/" PROJECTION = "full" + FIELDS = "items/contentLanguage,nextPageToken" EXPECTED = { "maxResults": 10, "prefix": PREFIX, "delimiter": DELIMITER, "projection": PROJECTION, + "fields": FIELDS, "userProject": USER_PROJECT, } connection = _Connection({"items": []}) @@ -817,6 +819,7 @@ def test_list_blob_objects_w_all_arguments_and_user_project(self): prefix=PREFIX, delimiter=DELIMITER, projection=PROJECTION, + fields=FIELDS, client=client, ) blobs = list(iterator) From 4d76d4858514cddf7442bec02345724e7f2a2e8d Mon Sep 17 00:00:00 2001 From: Sangram Date: Tue, 2 Jul 2019 01:41:13 +0530 Subject: [PATCH 5/7] Re-test system tests --- storage/tests/unit/test_bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index 84e47b6c6fe7..561f6ff19ddf 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -743,7 +743,7 @@ def test_list_blobs_defaults(self): self.assertEqual(kw["query_params"], {"projection": "noAcl"}) def test_list_blobs_objects_defaults(self): - NAME = "name" + NAME = "Name" connection = _Connection({"items": []}) client = _Client(connection) bucket = self._make_one(client=client, name=NAME) From 00b00f2172e00459ea7fb5b1d956526163afb002 Mon Sep 17 00:00:00 2001 From: Sangram Date: Fri, 5 Jul 2019 02:12:00 +0530 Subject: [PATCH 6/7] adding minor change to run CI after checkin in PR --- storage/tests/unit/test_bucket.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index 561f6ff19ddf..a04cbdeabbb2 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -757,7 +757,7 @@ def test_list_blobs_objects_defaults(self): def test_list_blobs_w_all_arguments_and_user_project(self): NAME = "name" - USER_PROJECT = "user-project-123" + USER_PROJECT = "user-project-1234" MAX_RESULTS = 10 PAGE_TOKEN = "ABCD" PREFIX = "subfolder" From e76625b46eee0ed3249b0e434103feb733eb564c Mon Sep 17 00:00:00 2001 From: Sangram Date: Sat, 10 Aug 2019 02:15:20 +0530 Subject: [PATCH 7/7] Move test below, remove default duplicate test --- storage/tests/unit/test_bucket.py | 37 ++++++++++--------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/storage/tests/unit/test_bucket.py b/storage/tests/unit/test_bucket.py index a04cbdeabbb2..d9ae460f371a 100644 --- a/storage/tests/unit/test_bucket.py +++ b/storage/tests/unit/test_bucket.py @@ -755,22 +755,18 @@ def test_list_blobs_objects_defaults(self): self.assertEqual(kw["path"], "/b/%s/o" % NAME) self.assertEqual(kw["query_params"], {"projection": "noAcl"}) - def test_list_blobs_w_all_arguments_and_user_project(self): + def test_list_blob_objects_w_all_arguments_and_user_project(self): NAME = "name" - USER_PROJECT = "user-project-1234" + USER_PROJECT = "user-project-123" MAX_RESULTS = 10 - PAGE_TOKEN = "ABCD" PREFIX = "subfolder" DELIMITER = "/" - VERSIONS = True PROJECTION = "full" FIELDS = "items/contentLanguage,nextPageToken" EXPECTED = { "maxResults": 10, - "pageToken": PAGE_TOKEN, "prefix": PREFIX, "delimiter": DELIMITER, - "versions": VERSIONS, "projection": PROJECTION, "fields": FIELDS, "userProject": USER_PROJECT, @@ -778,12 +774,10 @@ def test_list_blobs_w_all_arguments_and_user_project(self): connection = _Connection({"items": []}) client = _Client(connection) bucket = self._make_one(name=NAME, user_project=USER_PROJECT) - iterator = bucket.list_blobs( + iterator = bucket.list_blob_objects( max_results=MAX_RESULTS, - page_token=PAGE_TOKEN, prefix=PREFIX, delimiter=DELIMITER, - versions=VERSIONS, projection=PROJECTION, fields=FIELDS, client=client, @@ -795,18 +789,22 @@ def test_list_blobs_w_all_arguments_and_user_project(self): self.assertEqual(kw["path"], "/b/%s/o" % NAME) self.assertEqual(kw["query_params"], EXPECTED) - def test_list_blob_objects_w_all_arguments_and_user_project(self): + def test_list_blobs_w_all_arguments_and_user_project(self): NAME = "name" - USER_PROJECT = "user-project-123" + USER_PROJECT = "user-project-1234" MAX_RESULTS = 10 + PAGE_TOKEN = "ABCD" PREFIX = "subfolder" DELIMITER = "/" + VERSIONS = True PROJECTION = "full" FIELDS = "items/contentLanguage,nextPageToken" EXPECTED = { "maxResults": 10, + "pageToken": PAGE_TOKEN, "prefix": PREFIX, "delimiter": DELIMITER, + "versions": VERSIONS, "projection": PROJECTION, "fields": FIELDS, "userProject": USER_PROJECT, @@ -814,10 +812,12 @@ def test_list_blob_objects_w_all_arguments_and_user_project(self): connection = _Connection({"items": []}) client = _Client(connection) bucket = self._make_one(name=NAME, user_project=USER_PROJECT) - iterator = bucket.list_blob_objects( + iterator = bucket.list_blobs( max_results=MAX_RESULTS, + page_token=PAGE_TOKEN, prefix=PREFIX, delimiter=DELIMITER, + versions=VERSIONS, projection=PROJECTION, fields=FIELDS, client=client, @@ -829,19 +829,6 @@ def test_list_blob_objects_w_all_arguments_and_user_project(self): self.assertEqual(kw["path"], "/b/%s/o" % NAME) self.assertEqual(kw["query_params"], EXPECTED) - def test_list_blobs(self): - NAME = "name" - connection = _Connection({"items": []}) - client = _Client(connection) - bucket = self._make_one(client=client, name=NAME) - iterator = bucket.list_blobs() - blobs = list(iterator) - self.assertEqual(blobs, []) - kw, = connection._requested - self.assertEqual(kw["method"], "GET") - self.assertEqual(kw["path"], "/b/%s/o" % NAME) - self.assertEqual(kw["query_params"], {"projection": "noAcl"}) - def test_list_notifications(self): from google.cloud.storage.notification import BucketNotification from google.cloud.storage.notification import _TOPIC_REF_FMT